Compiler in Rust (lexer, parser, checker, C codegen), maps, chars, dino fmt, dino lsp (diagnostics, formatting, outline, definition, hover, completion, signature help), docs, examples, VS Code/Vim files. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
50 lines
1.3 KiB
Plaintext
50 lines
1.3 KiB
Plaintext
// Turns the text of a calculation into a list of tokens.
|
|
|
|
pub enum Kind {
|
|
Num
|
|
Name
|
|
Op
|
|
Open
|
|
Close
|
|
End
|
|
}
|
|
|
|
pub struct Token {
|
|
pub kind: Kind
|
|
pub text: string
|
|
pub pos: int
|
|
}
|
|
|
|
pub func tokenize(src: string): ([]Token, error) {
|
|
var tokens: []Token
|
|
var i = 0
|
|
for i < src.len() {
|
|
var c = src[i]
|
|
if c == ' ' {
|
|
i += 1
|
|
continue
|
|
}
|
|
var start = i
|
|
if c.is_digit() || c == '.' {
|
|
for i < src.len() && (src[i].is_digit() || src[i] == '.') { i += 1 }
|
|
tokens.push(Token{Kind.Num, src[start..i], start})
|
|
} else if c.is_letter() || c == '_' {
|
|
for i < src.len() && (src[i].is_alnum() || src[i] == '_') { i += 1 }
|
|
tokens.push(Token{Kind.Name, src[start..i], start})
|
|
} else if "+-*/^%=,".contains(string(c)) {
|
|
tokens.push(Token{Kind.Op, string(c), start})
|
|
i += 1
|
|
} else if c == '(' {
|
|
tokens.push(Token{Kind.Open, "(", start})
|
|
i += 1
|
|
} else if c == ')' {
|
|
tokens.push(Token{Kind.Close, ")", start})
|
|
i += 1
|
|
} else {
|
|
return tokens, error("unexpected `{c}` at column {i + 1}")
|
|
}
|
|
}
|
|
tokens.push(Token{Kind.End, "", src.len()})
|
|
return tokens, nil
|
|
}
|