Files
rexlang/examples/calc/token.rex
T
Your NameandClaude Opus 5.5 7cf9727bcd Rename the language from Dino to Rex
Files use .rex, the command is rex, and the crate, runtime prefixes,
docs, editor files and examples all use the new name.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-08 13:51:23 +01:00

50 lines
1.3 KiB
Rexx

// Turns the text of a calculation into a list of tokens.
pub enum Kind {
Num
Name
Op
Open
Close
End
}
pub struct Token {
pub kind: Kind
pub text: string
pub pos: int
}
pub func tokenize(src: string): ([]Token, error) {
var tokens: []Token
var i = 0
for i < src.len() {
var c = src[i]
if c == ' ' {
i += 1
continue
}
var start = i
if c.is_digit() || c == '.' {
for i < src.len() && (src[i].is_digit() || src[i] == '.') { i += 1 }
tokens.push(Token{Kind.Num, src[start..i], start})
} else if c.is_letter() || c == '_' {
for i < src.len() && (src[i].is_alnum() || src[i] == '_') { i += 1 }
tokens.push(Token{Kind.Name, src[start..i], start})
} else if "+-*/^%=,".contains(string(c)) {
tokens.push(Token{Kind.Op, string(c), start})
i += 1
} else if c == '(' {
tokens.push(Token{Kind.Open, "(", start})
i += 1
} else if c == ')' {
tokens.push(Token{Kind.Close, ")", start})
i += 1
} else {
return tokens, error("unexpected `{c}` at column {i + 1}")
}
}
tokens.push(Token{Kind.End, "", src.len()})
return tokens, nil
}