// Turns the text of a calculation into a list of tokens. pub enum Kind { Num Name Op Open Close End } pub struct Token { pub kind: Kind pub text: string pub pos: int } pub func tokenize(src: string): ([]Token, error) { var tokens: []Token var i = 0 for i < src.len() { var c = src[i] if c == ' ' { i += 1 continue } var start = i if c.is_digit() || c == '.' { for i < src.len() && (src[i].is_digit() || src[i] == '.') { i += 1 } tokens.push(Token{Kind.Num, src[start..i], start}) } else if c.is_letter() || c == '_' { for i < src.len() && (src[i].is_alnum() || src[i] == '_') { i += 1 } tokens.push(Token{Kind.Name, src[start..i], start}) } else if "+-*/^%=,".contains(string(c)) { tokens.push(Token{Kind.Op, string(c), start}) i += 1 } else if c == '(' { tokens.push(Token{Kind.Open, "(", start}) i += 1 } else if c == ')' { tokens.push(Token{Kind.Close, ")", start}) i += 1 } else { return tokens, error("unexpected `{c}` at column {i + 1}") } } tokens.push(Token{Kind.End, "", src.len()}) return tokens, nil }