diff --git a/src/cursor.rs b/src/cursor.rs new file mode 100644 index 0000000..84e26bf --- /dev/null +++ b/src/cursor.rs @@ -0,0 +1,19 @@ +pub struct Cursor { + pos: usize, +} + +impl Cursor { + pub fn pos(&self) -> usize { + self.pos + } + + pub fn advance_by(&mut self, n: usize) { + self.pos += n; + } +} + +impl Default for Cursor { + fn default() -> Self { + Cursor { pos: 0 } + } +} diff --git a/src/lexer.rs b/src/lexer.rs index c832d08..d61a36b 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,86 +1,103 @@ -#[derive(Debug, PartialEq, Eq)] -pub enum Token { - Asterisk, - Colon, - Comma, - CurlyBracketL, - CurlyBracketR, - Equals, - Fn, - GreaterThan, - Hyphen, - LessThan, - Ident, - Let, - Number, - ParenL, - ParenR, - Plus, - Semicolon, - Slash, - SquareBracketL, - SquareBracketR, - String, - Unknown, -} - -#[derive(Debug, PartialEq, Eq)] -pub struct Span<'a> { - start: usize, - raw: &'a str, -} - -impl<'a> Span<'a> { - fn new(start: usize, raw: &'a str) -> Self { - Span { start, raw } - } - - pub fn start(&self) -> usize { - self.start - } - - pub fn end(&self) -> usize { - self.start + self.raw.len() - } - - pub fn raw(&self) -> &'a str { - self.raw - } - - pub fn len(&self) -> usize { - self.raw.len() - } -} - -#[derive(Debug, PartialEq, Eq)] -pub struct TokenSpan<'a> { - pub token: Token, - pub span: Span<'a>, -} - -struct Cursor { - pos: usize, -} - -impl Cursor { - fn advance_by(&mut self, n: usize) { - self.pos += n; - } -} - -impl Default for Cursor { - fn default() -> Self { - Cursor { pos: 0 } - } -} +use crate::{ + cursor::Cursor, + span::{Span, Spanned}, + token::Token, +}; pub struct Lexer<'a> { input: &'a str, cursor: Cursor, } +impl<'a> Lexer<'a> { + pub fn new(input: &'a str) -> Self { + Lexer { + cursor: Cursor::default(), + input, + } + } + + fn consume(&mut self, n: usize) -> Span<'a> { + let start = self.cursor.pos(); + let end = start + n; + let span = Span::new(start, &self.input[start..end]); + self.cursor.advance_by(n); + span + } + + fn emit(&mut self, node: Token, len: usize) -> Spanned<'a, Token> { + Spanned::new(node, self.consume(len)) + } + + fn peek_char(&self) -> Option { + self.input[self.cursor.pos()..].chars().next() + } + + fn peek_ident(&self) -> Option<&'a str> { + let first = self + .peek_char() + .filter(|c| c.is_alphabetic() || *c == '_')?; + let start = self.cursor.pos(); + let remaining = start + first.len_utf8(); + + let offset = self.input[remaining..] + .char_indices() + .find(|(_, c)| !c.is_alphanumeric() && *c != '_') + .map(|(idx, _)| idx) + .unwrap_or(self.input[remaining..].len()); + + Some(&self.input[start..remaining + offset]) + } + + fn peek_number(&self) -> Option<&'a str> { + let first = self.peek_char().filter(|c| c.is_digit(10))?; + let start = self.cursor.pos(); + let remaining = start + first.len_utf8(); + + let offset = self.input[remaining..] + .char_indices() + .find(|(_, c)| !c.is_digit(10) && *c != '_' && *c != '.') + .map(|(idx, _)| idx) + .unwrap_or(self.input[remaining..].len()); + + Some(&self.input[start..remaining + offset]) + } + + fn peek_string(&self) -> Option<&'a str> { + let first = self.peek_char().filter(|c| *c == '"')?; + let start = self.cursor.pos(); + let remaining = start + first.len_utf8(); + + let mut escaped = false; + let end = self.input[remaining..] + .char_indices() + .find(|&(_, c)| match (escaped, c) { + (true, _) => { + escaped = false; + false + } + (false, '\\') => { + escaped = true; + false + } + (false, '"') => true, + (false, _) => false, + }) + .map(|(idx, c)| remaining + idx + c.len_utf8()) + .unwrap_or(self.input.len()); + + Some(&self.input[start..end]) + } + + fn skip_whitespace(&mut self) { + while let Some(c) = self.peek_char().filter(|c| c.is_whitespace()) { + self.cursor.advance_by(c.len_utf8()); + } + } +} + impl<'a> Iterator for Lexer<'a> { - type Item = TokenSpan<'a>; + type Item = Spanned<'a, Token>; fn next(&mut self) -> Option { self.skip_whitespace(); @@ -129,96 +146,6 @@ impl<'a> Iterator for Lexer<'a> { } } -impl<'a> Lexer<'a> { - pub fn new(input: &'a str) -> Self { - Lexer { - cursor: Cursor::default(), - input, - } - } - - fn consume(&mut self, n: usize) -> Span<'a> { - let start = self.cursor.pos; - let end = start + n; - let span = Span::new(start, &self.input[start..end]); - self.cursor.advance_by(n); - span - } - - fn emit(&mut self, token: Token, len: usize) -> TokenSpan<'a> { - TokenSpan { - token, - span: self.consume(len), - } - } - - fn peek_char(&self) -> Option { - self.input[self.cursor.pos..].chars().next() - } - - fn peek_ident(&self) -> Option<&'a str> { - let first = self - .peek_char() - .filter(|c| c.is_alphabetic() || *c == '_')?; - let start = self.cursor.pos; - let remaining = start + first.len_utf8(); - - let offset = self.input[remaining..] - .char_indices() - .find(|(_, c)| !c.is_alphanumeric() && *c != '_') - .map(|(idx, _)| idx) - .unwrap_or(self.input[remaining..].len()); - - Some(&self.input[start..remaining + offset]) - } - - fn peek_number(&self) -> Option<&'a str> { - let first = self.peek_char().filter(|c| c.is_digit(10))?; - let start = self.cursor.pos; - let remaining = start + first.len_utf8(); - - let offset = self.input[remaining..] - .char_indices() - .find(|(_, c)| !c.is_digit(10) && *c != '_' && *c != '.') - .map(|(idx, _)| idx) - .unwrap_or(self.input[remaining..].len()); - - Some(&self.input[start..remaining + offset]) - } - - fn peek_string(&self) -> Option<&'a str> { - let first = self.peek_char().filter(|c| *c == '"')?; - let start = self.cursor.pos; - let remaining = start + first.len_utf8(); - - let mut escaped = false; - let end = self.input[remaining..] - .char_indices() - .find(|&(_, c)| match (escaped, c) { - (true, _) => { - escaped = false; - false - } - (false, '\\') => { - escaped = true; - false - } - (false, '"') => true, - (false, _) => false, - }) - .map(|(idx, c)| remaining + idx + c.len_utf8()) - .unwrap_or(self.input.len()); - - Some(&self.input[start..end]) - } - - fn skip_whitespace(&mut self) { - while let Some(c) = self.peek_char().filter(|c| c.is_whitespace()) { - self.cursor.advance_by(c.len_utf8()); - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -277,13 +204,10 @@ mod tests { assert_eq!(lexer.next(), None); } - fn assert_next_token(lexer: &mut Lexer, expected: Token, raw: &str, offset: usize) { + fn assert_next_token(lexer: &mut Lexer, node: Token, raw: &str, offset: usize) { assert_eq!( lexer.next(), - Some(TokenSpan { - token: expected, - span: Span::new(offset, raw), - }) + Some(Spanned::new(node, Span::new(offset, raw))) ); } } diff --git a/src/lib.rs b/src/lib.rs index c3f86b1..3a33966 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1,5 @@ pub mod ast; +pub mod cursor; pub mod lexer; +pub mod span; +pub mod token; diff --git a/src/span.rs b/src/span.rs new file mode 100644 index 0000000..1fb0145 --- /dev/null +++ b/src/span.rs @@ -0,0 +1,47 @@ +#[derive(Debug, PartialEq, Eq)] +pub struct Span<'a> { + start: usize, + raw: &'a str, +} + +impl<'a> Span<'a> { + pub fn new(start: usize, raw: &'a str) -> Self { + Span { start, raw } + } + + pub fn start(&self) -> usize { + self.start + } + + pub fn raw(&self) -> &'a str { + self.raw + } + + pub fn len(&self) -> usize { + self.raw.len() + } + + pub fn end(&self) -> usize { + self.start + self.raw.len() + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Spanned<'a, T> { + node: T, + span: Span<'a>, +} + +impl<'a, T> Spanned<'a, T> { + pub fn new(node: T, span: Span<'a>) -> Self { + Spanned { node, span } + } + + pub fn node(&self) -> &T { + &self.node + } + + pub fn span(&self) -> &Span<'a> { + &self.span + } +} diff --git a/src/token.rs b/src/token.rs new file mode 100644 index 0000000..a6d3611 --- /dev/null +++ b/src/token.rs @@ -0,0 +1,25 @@ +#[derive(Debug, PartialEq, Eq)] +pub enum Token { + Asterisk, + Colon, + Comma, + CurlyBracketL, + CurlyBracketR, + Equals, + Fn, + GreaterThan, + Hyphen, + LessThan, + Ident, + Let, + Number, + ParenL, + ParenR, + Plus, + Semicolon, + Slash, + SquareBracketL, + SquareBracketR, + String, + Unknown, +}