From 964820a780571766404fa8142ea31d9b9de572f6 Mon Sep 17 00:00:00 2001 From: Rogier Reerink Date: Sun, 15 Nov 2026 11:59:16 +0100 Subject: [PATCH] Fixed some stuff and unquoted/unescaped strings. --- src/parser.rs | 123 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 99 insertions(+), 24 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 2f1cdcb..64b0794 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -32,7 +32,9 @@ where } fn next(&mut self) -> Option> { - self.tokens.next() + let next = self.tokens.next()?; + self.current_span = *next.span(); + Some(next) } fn next_expected(&mut self, expected: Token) -> Result, ParseError> { @@ -80,12 +82,11 @@ where match self.peek() { Some(Token::Let) => { let r#let = self.next().unwrap(); - let ident = self.next_expected(Token::Ident).map(|spanned| { - Ok(Ident { - name: self.slice(*spanned.span())?.to_string(), - span: *spanned.span(), - }) - })??; + let ident = self.next_expected(Token::Ident)?; + let ident = Ident { + name: self.slice(*ident.span())?.to_string(), + span: *ident.span(), + }; self.next_expected(Token::Equals)?; let expr = self.parse_expr()?; let semi = self.next_expected(Token::Semicolon)?; @@ -118,12 +119,14 @@ where match self.peek() { Some(Token::Number) => { let token = self.next().unwrap(); - let raw = self.slice(*token.span())?; let number = LitNumber { - value: raw.parse::().map_err(|e| ParseError { - span: *token.span(), - message: format!("Failed to parse number: {}", e), - })?, + value: self + .slice(*token.span())? + .parse::() + .map_err(|e| ParseError { + span: *token.span(), + message: format!("Failed to parse number: {}", e), + })?, span: *token.span(), }; Ok(Expr::Literal(number.into())) @@ -131,17 +134,18 @@ where Some(Token::String) => { let token = self.next().unwrap(); let raw = self.slice(*token.span())?; + let unquoted = Self::unquote_string(raw, token.span().start())?; + let escaped = Self::unescape_string(unquoted, token.span().start() + 1)?; let string = LitString { - value: raw[1..raw.len() - 1].to_string(), + value: escaped, span: *token.span(), }; Ok(Expr::Literal(string.into())) } Some(Token::Ident) => { let token = self.next().unwrap(); - let raw = self.slice(*token.span())?; let ident = Ident { - name: raw.to_string(), + name: self.slice(*token.span())?.to_string(), span: *token.span(), }; match self.peek() { @@ -150,7 +154,9 @@ where let mut args = Vec::new(); while self.peek() != Some(&Token::ParenR) { args.push(self.parse_expr()?); - self.next_if(Token::Comma); + if self.next_if(Token::Comma).is_none() { + break; + } } let rparen = self.next_expected(Token::ParenR)?; let fncall = FnCall { @@ -173,6 +179,48 @@ where }), } } + + fn unquote_string<'s>(s: &'s str, offset: usize) -> Result<&'s str, ParseError> { + Ok(s.strip_prefix('"') + .and_then(|s| s.strip_suffix('"')) + .ok_or_else(|| ParseError { + span: Span::new(offset, offset + s.len()), + message: "Malformed string literal: missing quotes".to_string(), + })?) + } + + fn unescape_string(s: &str, offset: usize) -> Result { + let mut result = String::new(); + let mut chars = s.char_indices(); + while let Some((_, c)) = chars.next() { + if c == '\\' { + match chars.next() { + Some((_, 'n')) => result.push('\n'), + Some((_, 't')) => result.push('\t'), + Some((_, 'r')) => result.push('\r'), + Some((_, '"')) => result.push('"'), + Some((_, '\\')) => result.push('\\'), + Some((i, other)) => { + let start = offset + i; + return Err(ParseError { + span: Span::new(start, start + other.len_utf8()), + message: format!("Unknown escape sequence: \\{}", other), + }); + } + None => { + let start = offset + s.len(); + return Err(ParseError { + span: Span::new(start, start), + message: "Incomplete escape sequence".to_string(), + }); + } + } + } else { + result.push(c); + } + } + Ok(result) + } } #[derive(Debug, PartialEq)] @@ -198,7 +246,7 @@ mod tests { use super::*; #[test] - fn stmt_let() { + fn stmt_lets() { let input = "let x = 500.0;"; let lexer = Lexer::new(input); let mut parser = Parser::new(input, lexer); @@ -267,7 +315,7 @@ mod tests { } #[test] - fn expr_number() { + fn expr_numbers() { let input = "500.0"; let lexer = Lexer::new(input); let mut parser = Parser::new(input, lexer); @@ -285,16 +333,16 @@ mod tests { } #[test] - fn expr_string() { - let input = "\"hello\""; + fn expr_strings() { + let input = "\"hello\\n\\t\\\\hey\""; let lexer = Lexer::new(input); let mut parser = Parser::new(input, lexer); pretty_assertions::assert_eq!( parser.parse_expr(), Ok(Expr::Literal( LitString { - value: "hello".to_string(), - span: Span::new(0, 7) + value: "hello\n\t\\hey".to_string(), + span: Span::new(0, 16) } .into() )) @@ -303,7 +351,34 @@ mod tests { } #[test] - fn expr_variable() { + fn expr_string_errors() { + let input = "\"hello"; + let lexer = Lexer::new(input); + let mut parser = Parser::new(input, lexer); + pretty_assertions::assert_eq!( + parser.parse_expr(), + Err(ParseError { + span: Span::new(0, 6), + message: "Malformed string literal: missing quotes".to_string() + }) + ); + pretty_assertions::assert_eq!(parser.next(), None); + + let input = "\"hello \\\""; + let lexer = Lexer::new(input); + let mut parser = Parser::new(input, lexer); + pretty_assertions::assert_eq!( + parser.parse_expr(), + Err(ParseError { + span: Span::new(8, 8), + message: "Incomplete escape sequence".to_string() + }) + ); + pretty_assertions::assert_eq!(parser.next(), None); + } + + #[test] + fn expr_variables() { let input = "x"; let lexer = Lexer::new(input); let mut parser = Parser::new(input, lexer); @@ -318,7 +393,7 @@ mod tests { } #[test] - fn expr_fncall() { + fn expr_fncalls() { let input = "f(x)"; let lexer = Lexer::new(input); let mut parser = Parser::new(input, lexer);