Organising.

This commit is contained in:
2026-11-10 21:46:01 +01:00
parent 5ed7b2c6c8
commit 3fca1be67b
5 changed files with 189 additions and 171 deletions
+19
View File
@@ -0,0 +1,19 @@
pub struct Cursor {
pos: usize,
}
impl Cursor {
pub fn pos(&self) -> usize {
self.pos
}
pub fn advance_by(&mut self, n: usize) {
self.pos += n;
}
}
impl Default for Cursor {
fn default() -> Self {
Cursor { pos: 0 }
}
}
+95 -171
View File
@@ -1,86 +1,103 @@
#[derive(Debug, PartialEq, Eq)] use crate::{
pub enum Token { cursor::Cursor,
Asterisk, span::{Span, Spanned},
Colon, token::Token,
Comma, };
CurlyBracketL,
CurlyBracketR,
Equals,
Fn,
GreaterThan,
Hyphen,
LessThan,
Ident,
Let,
Number,
ParenL,
ParenR,
Plus,
Semicolon,
Slash,
SquareBracketL,
SquareBracketR,
String,
Unknown,
}
#[derive(Debug, PartialEq, Eq)]
pub struct Span<'a> {
start: usize,
raw: &'a str,
}
impl<'a> Span<'a> {
fn new(start: usize, raw: &'a str) -> Self {
Span { start, raw }
}
pub fn start(&self) -> usize {
self.start
}
pub fn end(&self) -> usize {
self.start + self.raw.len()
}
pub fn raw(&self) -> &'a str {
self.raw
}
pub fn len(&self) -> usize {
self.raw.len()
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct TokenSpan<'a> {
pub token: Token,
pub span: Span<'a>,
}
struct Cursor {
pos: usize,
}
impl Cursor {
fn advance_by(&mut self, n: usize) {
self.pos += n;
}
}
impl Default for Cursor {
fn default() -> Self {
Cursor { pos: 0 }
}
}
pub struct Lexer<'a> { pub struct Lexer<'a> {
input: &'a str, input: &'a str,
cursor: Cursor, cursor: Cursor,
} }
impl<'a> Lexer<'a> {
pub fn new(input: &'a str) -> Self {
Lexer {
cursor: Cursor::default(),
input,
}
}
fn consume(&mut self, n: usize) -> Span<'a> {
let start = self.cursor.pos();
let end = start + n;
let span = Span::new(start, &self.input[start..end]);
self.cursor.advance_by(n);
span
}
fn emit(&mut self, node: Token, len: usize) -> Spanned<'a, Token> {
Spanned::new(node, self.consume(len))
}
fn peek_char(&self) -> Option<char> {
self.input[self.cursor.pos()..].chars().next()
}
fn peek_ident(&self) -> Option<&'a str> {
let first = self
.peek_char()
.filter(|c| c.is_alphabetic() || *c == '_')?;
let start = self.cursor.pos();
let remaining = start + first.len_utf8();
let offset = self.input[remaining..]
.char_indices()
.find(|(_, c)| !c.is_alphanumeric() && *c != '_')
.map(|(idx, _)| idx)
.unwrap_or(self.input[remaining..].len());
Some(&self.input[start..remaining + offset])
}
fn peek_number(&self) -> Option<&'a str> {
let first = self.peek_char().filter(|c| c.is_digit(10))?;
let start = self.cursor.pos();
let remaining = start + first.len_utf8();
let offset = self.input[remaining..]
.char_indices()
.find(|(_, c)| !c.is_digit(10) && *c != '_' && *c != '.')
.map(|(idx, _)| idx)
.unwrap_or(self.input[remaining..].len());
Some(&self.input[start..remaining + offset])
}
fn peek_string(&self) -> Option<&'a str> {
let first = self.peek_char().filter(|c| *c == '"')?;
let start = self.cursor.pos();
let remaining = start + first.len_utf8();
let mut escaped = false;
let end = self.input[remaining..]
.char_indices()
.find(|&(_, c)| match (escaped, c) {
(true, _) => {
escaped = false;
false
}
(false, '\\') => {
escaped = true;
false
}
(false, '"') => true,
(false, _) => false,
})
.map(|(idx, c)| remaining + idx + c.len_utf8())
.unwrap_or(self.input.len());
Some(&self.input[start..end])
}
fn skip_whitespace(&mut self) {
while let Some(c) = self.peek_char().filter(|c| c.is_whitespace()) {
self.cursor.advance_by(c.len_utf8());
}
}
}
impl<'a> Iterator for Lexer<'a> { impl<'a> Iterator for Lexer<'a> {
type Item = TokenSpan<'a>; type Item = Spanned<'a, Token>;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
self.skip_whitespace(); self.skip_whitespace();
@@ -129,96 +146,6 @@ impl<'a> Iterator for Lexer<'a> {
} }
} }
impl<'a> Lexer<'a> {
pub fn new(input: &'a str) -> Self {
Lexer {
cursor: Cursor::default(),
input,
}
}
fn consume(&mut self, n: usize) -> Span<'a> {
let start = self.cursor.pos;
let end = start + n;
let span = Span::new(start, &self.input[start..end]);
self.cursor.advance_by(n);
span
}
fn emit(&mut self, token: Token, len: usize) -> TokenSpan<'a> {
TokenSpan {
token,
span: self.consume(len),
}
}
fn peek_char(&self) -> Option<char> {
self.input[self.cursor.pos..].chars().next()
}
fn peek_ident(&self) -> Option<&'a str> {
let first = self
.peek_char()
.filter(|c| c.is_alphabetic() || *c == '_')?;
let start = self.cursor.pos;
let remaining = start + first.len_utf8();
let offset = self.input[remaining..]
.char_indices()
.find(|(_, c)| !c.is_alphanumeric() && *c != '_')
.map(|(idx, _)| idx)
.unwrap_or(self.input[remaining..].len());
Some(&self.input[start..remaining + offset])
}
fn peek_number(&self) -> Option<&'a str> {
let first = self.peek_char().filter(|c| c.is_digit(10))?;
let start = self.cursor.pos;
let remaining = start + first.len_utf8();
let offset = self.input[remaining..]
.char_indices()
.find(|(_, c)| !c.is_digit(10) && *c != '_' && *c != '.')
.map(|(idx, _)| idx)
.unwrap_or(self.input[remaining..].len());
Some(&self.input[start..remaining + offset])
}
fn peek_string(&self) -> Option<&'a str> {
let first = self.peek_char().filter(|c| *c == '"')?;
let start = self.cursor.pos;
let remaining = start + first.len_utf8();
let mut escaped = false;
let end = self.input[remaining..]
.char_indices()
.find(|&(_, c)| match (escaped, c) {
(true, _) => {
escaped = false;
false
}
(false, '\\') => {
escaped = true;
false
}
(false, '"') => true,
(false, _) => false,
})
.map(|(idx, c)| remaining + idx + c.len_utf8())
.unwrap_or(self.input.len());
Some(&self.input[start..end])
}
fn skip_whitespace(&mut self) {
while let Some(c) = self.peek_char().filter(|c| c.is_whitespace()) {
self.cursor.advance_by(c.len_utf8());
}
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -277,13 +204,10 @@ mod tests {
assert_eq!(lexer.next(), None); assert_eq!(lexer.next(), None);
} }
fn assert_next_token(lexer: &mut Lexer, expected: Token, raw: &str, offset: usize) { fn assert_next_token(lexer: &mut Lexer, node: Token, raw: &str, offset: usize) {
assert_eq!( assert_eq!(
lexer.next(), lexer.next(),
Some(TokenSpan { Some(Spanned::new(node, Span::new(offset, raw)))
token: expected,
span: Span::new(offset, raw),
})
); );
} }
} }
+3
View File
@@ -1,2 +1,5 @@
pub mod ast; pub mod ast;
pub mod cursor;
pub mod lexer; pub mod lexer;
pub mod span;
pub mod token;
+47
View File
@@ -0,0 +1,47 @@
#[derive(Debug, PartialEq, Eq)]
pub struct Span<'a> {
start: usize,
raw: &'a str,
}
impl<'a> Span<'a> {
pub fn new(start: usize, raw: &'a str) -> Self {
Span { start, raw }
}
pub fn start(&self) -> usize {
self.start
}
pub fn raw(&self) -> &'a str {
self.raw
}
pub fn len(&self) -> usize {
self.raw.len()
}
pub fn end(&self) -> usize {
self.start + self.raw.len()
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct Spanned<'a, T> {
node: T,
span: Span<'a>,
}
impl<'a, T> Spanned<'a, T> {
pub fn new(node: T, span: Span<'a>) -> Self {
Spanned { node, span }
}
pub fn node(&self) -> &T {
&self.node
}
pub fn span(&self) -> &Span<'a> {
&self.span
}
}
+25
View File
@@ -0,0 +1,25 @@
#[derive(Debug, PartialEq, Eq)]
pub enum Token {
Asterisk,
Colon,
Comma,
CurlyBracketL,
CurlyBracketR,
Equals,
Fn,
GreaterThan,
Hyphen,
LessThan,
Ident,
Let,
Number,
ParenL,
ParenR,
Plus,
Semicolon,
Slash,
SquareBracketL,
SquareBracketR,
String,
Unknown,
}