Fixed some stuff and unquoted/escaped strings.

This commit is contained in:
2026-11-15 11:59:16 +01:00
parent 8072a9f915
commit 21e101b16f
+96 -21
View File
@@ -32,7 +32,9 @@ where
} }
fn next(&mut self) -> Option<Spanned<Token>> { fn next(&mut self) -> Option<Spanned<Token>> {
self.tokens.next() let next = self.tokens.next()?;
self.current_span = *next.span();
Some(next)
} }
fn next_expected(&mut self, expected: Token) -> Result<Spanned<Token>, ParseError> { fn next_expected(&mut self, expected: Token) -> Result<Spanned<Token>, ParseError> {
@@ -80,12 +82,11 @@ where
match self.peek() { match self.peek() {
Some(Token::Let) => { Some(Token::Let) => {
let r#let = self.next().unwrap(); let r#let = self.next().unwrap();
let ident = self.next_expected(Token::Ident).map(|spanned| { let ident = self.next_expected(Token::Ident)?;
Ok(Ident { let ident = Ident {
name: self.slice(*spanned.span())?.to_string(), name: self.slice(*ident.span())?.to_string(),
span: *spanned.span(), span: *ident.span(),
}) };
})??;
self.next_expected(Token::Equals)?; self.next_expected(Token::Equals)?;
let expr = self.parse_expr()?; let expr = self.parse_expr()?;
let semi = self.next_expected(Token::Semicolon)?; let semi = self.next_expected(Token::Semicolon)?;
@@ -118,9 +119,11 @@ where
match self.peek() { match self.peek() {
Some(Token::Number) => { Some(Token::Number) => {
let token = self.next().unwrap(); let token = self.next().unwrap();
let raw = self.slice(*token.span())?;
let number = LitNumber { let number = LitNumber {
value: raw.parse::<f64>().map_err(|e| ParseError { value: self
.slice(*token.span())?
.parse::<f64>()
.map_err(|e| ParseError {
span: *token.span(), span: *token.span(),
message: format!("Failed to parse number: {}", e), message: format!("Failed to parse number: {}", e),
})?, })?,
@@ -131,17 +134,18 @@ where
Some(Token::String) => { Some(Token::String) => {
let token = self.next().unwrap(); let token = self.next().unwrap();
let raw = self.slice(*token.span())?; let raw = self.slice(*token.span())?;
let unquoted = Self::unquote_string(raw, token.span().start())?;
let escaped = Self::unescape_string(unquoted, token.span().start() + 1)?;
let string = LitString { let string = LitString {
value: raw[1..raw.len() - 1].to_string(), value: escaped,
span: *token.span(), span: *token.span(),
}; };
Ok(Expr::Literal(string.into())) Ok(Expr::Literal(string.into()))
} }
Some(Token::Ident) => { Some(Token::Ident) => {
let token = self.next().unwrap(); let token = self.next().unwrap();
let raw = self.slice(*token.span())?;
let ident = Ident { let ident = Ident {
name: raw.to_string(), name: self.slice(*token.span())?.to_string(),
span: *token.span(), span: *token.span(),
}; };
match self.peek() { match self.peek() {
@@ -150,7 +154,9 @@ where
let mut args = Vec::new(); let mut args = Vec::new();
while self.peek() != Some(&Token::ParenR) { while self.peek() != Some(&Token::ParenR) {
args.push(self.parse_expr()?); args.push(self.parse_expr()?);
self.next_if(Token::Comma); if self.next_if(Token::Comma).is_none() {
break;
}
} }
let rparen = self.next_expected(Token::ParenR)?; let rparen = self.next_expected(Token::ParenR)?;
let fncall = FnCall { let fncall = FnCall {
@@ -173,6 +179,48 @@ where
}), }),
} }
} }
fn unquote_string<'s>(s: &'s str, offset: usize) -> Result<&'s str, ParseError> {
Ok(s.strip_prefix('"')
.and_then(|s| s.strip_suffix('"'))
.ok_or_else(|| ParseError {
span: Span::new(offset, offset + s.len()),
message: "Malformed string literal: missing quotes".to_string(),
})?)
}
fn unescape_string(s: &str, offset: usize) -> Result<String, ParseError> {
let mut result = String::new();
let mut chars = s.char_indices();
while let Some((_, c)) = chars.next() {
if c == '\\' {
match chars.next() {
Some((_, 'n')) => result.push('\n'),
Some((_, 't')) => result.push('\t'),
Some((_, 'r')) => result.push('\r'),
Some((_, '"')) => result.push('"'),
Some((_, '\\')) => result.push('\\'),
Some((i, other)) => {
let start = offset + i;
return Err(ParseError {
span: Span::new(start, start + other.len_utf8()),
message: format!("Unknown escape sequence: \\{}", other),
});
}
None => {
let start = offset + s.len();
return Err(ParseError {
span: Span::new(start, start),
message: "Incomplete escape sequence".to_string(),
});
}
}
} else {
result.push(c);
}
}
Ok(result)
}
} }
#[derive(Debug, PartialEq)] #[derive(Debug, PartialEq)]
@@ -198,7 +246,7 @@ mod tests {
use super::*; use super::*;
#[test] #[test]
fn stmt_let() { fn stmt_lets() {
let input = "let x = 500.0;"; let input = "let x = 500.0;";
let lexer = Lexer::new(input); let lexer = Lexer::new(input);
let mut parser = Parser::new(input, lexer); let mut parser = Parser::new(input, lexer);
@@ -267,7 +315,7 @@ mod tests {
} }
#[test] #[test]
fn expr_number() { fn expr_numbers() {
let input = "500.0"; let input = "500.0";
let lexer = Lexer::new(input); let lexer = Lexer::new(input);
let mut parser = Parser::new(input, lexer); let mut parser = Parser::new(input, lexer);
@@ -285,16 +333,16 @@ mod tests {
} }
#[test] #[test]
fn expr_string() { fn expr_strings() {
let input = "\"hello\""; let input = "\"hello\\n\\t\\\\hey\"";
let lexer = Lexer::new(input); let lexer = Lexer::new(input);
let mut parser = Parser::new(input, lexer); let mut parser = Parser::new(input, lexer);
pretty_assertions::assert_eq!( pretty_assertions::assert_eq!(
parser.parse_expr(), parser.parse_expr(),
Ok(Expr::Literal( Ok(Expr::Literal(
LitString { LitString {
value: "hello".to_string(), value: "hello\n\t\\hey".to_string(),
span: Span::new(0, 7) span: Span::new(0, 16)
} }
.into() .into()
)) ))
@@ -303,7 +351,34 @@ mod tests {
} }
#[test] #[test]
fn expr_variable() { fn expr_string_errors() {
let input = "\"hello";
let lexer = Lexer::new(input);
let mut parser = Parser::new(input, lexer);
pretty_assertions::assert_eq!(
parser.parse_expr(),
Err(ParseError {
span: Span::new(0, 6),
message: "Malformed string literal: missing quotes".to_string()
})
);
pretty_assertions::assert_eq!(parser.next(), None);
let input = "\"hello \\\"";
let lexer = Lexer::new(input);
let mut parser = Parser::new(input, lexer);
pretty_assertions::assert_eq!(
parser.parse_expr(),
Err(ParseError {
span: Span::new(8, 8),
message: "Incomplete escape sequence".to_string()
})
);
pretty_assertions::assert_eq!(parser.next(), None);
}
#[test]
fn expr_variables() {
let input = "x"; let input = "x";
let lexer = Lexer::new(input); let lexer = Lexer::new(input);
let mut parser = Parser::new(input, lexer); let mut parser = Parser::new(input, lexer);
@@ -318,7 +393,7 @@ mod tests {
} }
#[test] #[test]
fn expr_fncall() { fn expr_fncalls() {
let input = "f(x)"; let input = "f(x)";
let lexer = Lexer::new(input); let lexer = Lexer::new(input);
let mut parser = Parser::new(input, lexer); let mut parser = Parser::new(input, lexer);