Dino compiler: language, formatter, language server, docs, editor support
Compiler in Rust (lexer, parser, checker, C codegen), maps, chars, dino fmt, dino lsp (diagnostics, formatting, outline, definition, hover, completion, signature help), docs, examples, VS Code/Vim files. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3e2779f447
commit
eaf9567a36
117 files changed
+19970
No files matched your search
@@ -0,0 +1,49 @@
|
||||
// Turns the text of a calculation into a list of tokens.
|
||||
|
||||
pub enum Kind {
|
||||
Num
|
||||
Name
|
||||
Op
|
||||
Open
|
||||
Close
|
||||
End
|
||||
}
|
||||
|
||||
pub struct Token {
|
||||
pub kind: Kind
|
||||
pub text: string
|
||||
pub pos: int
|
||||
}
|
||||
|
||||
pub func tokenize(src: string): ([]Token, error) {
|
||||
var tokens: []Token
|
||||
var i = 0
|
||||
for i < src.len() {
|
||||
var c = src[i]
|
||||
if c == ' ' {
|
||||
i += 1
|
||||
continue
|
||||
}
|
||||
var start = i
|
||||
if c.is_digit() || c == '.' {
|
||||
for i < src.len() && (src[i].is_digit() || src[i] == '.') { i += 1 }
|
||||
tokens.push(Token{Kind.Num, src[start..i], start})
|
||||
} else if c.is_letter() || c == '_' {
|
||||
for i < src.len() && (src[i].is_alnum() || src[i] == '_') { i += 1 }
|
||||
tokens.push(Token{Kind.Name, src[start..i], start})
|
||||
} else if "+-*/^%=,".contains(string(c)) {
|
||||
tokens.push(Token{Kind.Op, string(c), start})
|
||||
i += 1
|
||||
} else if c == '(' {
|
||||
tokens.push(Token{Kind.Open, "(", start})
|
||||
i += 1
|
||||
} else if c == ')' {
|
||||
tokens.push(Token{Kind.Close, ")", start})
|
||||
i += 1
|
||||
} else {
|
||||
return tokens, error("unexpected `{c}` at column {i + 1}")
|
||||
}
|
||||
}
|
||||
tokens.push(Token{Kind.End, "", src.len()})
|
||||
return tokens, nil
|
||||
}
|
||||
Reference in new issue
Block a user