Files
rexlang/src/fmt.rs
T
Your NameandClaude Sonnet 5.5 910bde692a Import files from folders above: import ..a.b, import ...a.b
Leading dots go up one folder less than their count (.. is one up, ...
is two). The language server index and formatter understand them, and a
missing file is reported with its real path.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-08 23:54:11 +01:00

463 lines
16 KiB
Rust

//! The code formatter behind `rex fmt`.
//!
//! It works on the text, not on the syntax tree, so comments and blank lines survive. It
//! never moves a line break: it fixes the indentation, the spaces between tokens and runs of
//! blank lines. Before handing back the result it lexes both versions and checks that they
//! hold exactly the same tokens, so a formatter bug can never change a program.
use crate::error::Diag;
use crate::lexer::{self, StrPart, Tok};
const INDENT: &str = " ";
#[derive(Clone, Copy, PartialEq, Debug)]
enum Kind {
Word,
Num,
Str,
Char,
Punct,
LineComment,
BlockComment,
}
#[derive(Clone, Debug)]
struct Piece {
kind: Kind,
text: String,
/// How many line breaks come before this piece (0: same line).
newlines: usize,
}
const KEYWORDS: &[&str] = &[
"import", "struct", "enum", "interface", "func", "var", "const", "if", "else", "for", "in", "match", "return",
"break", "continue", "pub", "shared", "weak", "step",
];
/// Words that make a following `{` open a block (rather than a struct or map literal).
const BLOCK_WORDS: &[&str] = &["func", "if", "for", "match", "struct", "enum", "interface", "else", "import"];
const BINARY: &[&str] = &[
"+", "-", "*", "/", "%", "==", "!=", "<", ">", "<=", ">=", "&&", "||", "=", "+=", "-=", "*=", "/=", "%=", "=>",
];
/// Format a whole file. Fails (with the problem) if the file doesn't parse.
pub fn format(src: &str) -> Result<String, Diag> {
crate::parser::parse(src)?;
let pieces = scan(src);
let out = print(&pieces);
// the safety net: the same tokens in, the same tokens out
let before = fingerprint(src)?;
let after = fingerprint(&out).map_err(|d| {
Diag::new(format!("the formatter produced code that doesn't lex ({}); this is a bug in `rex fmt`", d.msg), d.span)
})?;
if before != after {
return Err(Diag::new(
"the formatter would change the meaning of this file; this is a bug in `rex fmt`, nothing was written",
crate::ast::Span::default(),
));
}
Ok(out)
}
/// The tokens of `src`, without positions and without repeated line ends.
fn fingerprint(src: &str) -> Result<Vec<String>, Diag> {
let mut out: Vec<String> = Vec::new();
for t in lexer::lex(src)? {
let s = match &t.tok {
Tok::Str(parts) => {
let mut s = String::from("str:");
for p in parts {
match p {
StrPart::Text(t) => s.push_str(&format!("T{t:?}")),
StrPart::Code { src, .. } => s.push_str(&format!("C{src:?}")),
}
}
s
}
Tok::Newline if out.last().map_or(true, |l| l == "Newline") => continue,
other => format!("{other:?}"),
};
out.push(s);
}
Ok(out)
}
// ---------- reading the text ----------
fn string_end(c: &[char], mut i: usize) -> usize {
i += 1; // opening quote
loop {
match c.get(i) {
None => return i,
Some('\\') => i += 2,
Some('"') => return i + 1,
Some('{') => {
let mut depth = 1;
i += 1;
while depth > 0 {
match c.get(i) {
None => return i,
Some('"') => i = string_end(c, i),
Some('{') => {
depth += 1;
i += 1;
}
Some('}') => {
depth -= 1;
i += 1;
}
_ => i += 1,
}
}
}
_ => i += 1,
}
}
}
fn scan(src: &str) -> Vec<Piece> {
let c: Vec<char> = src.chars().collect();
let mut out: Vec<Piece> = Vec::new();
let mut i = 0;
let mut newlines = 0;
let take = |from: usize, to: usize| -> String { c[from..to.min(c.len())].iter().collect() };
while i < c.len() {
let ch = c[i];
if ch == '\n' {
newlines += 1;
i += 1;
continue;
}
if ch.is_whitespace() {
i += 1;
continue;
}
let start = i;
let kind;
if ch == '/' && c.get(i + 1) == Some(&'/') {
while i < c.len() && c[i] != '\n' {
i += 1;
}
kind = Kind::LineComment;
} else if ch == '/' && c.get(i + 1) == Some(&'*') {
i += 2;
while i < c.len() && !(c[i] == '*' && c.get(i + 1) == Some(&'/')) {
i += 1;
}
i = (i + 2).min(c.len());
kind = Kind::BlockComment;
} else if ch == '"' {
i = string_end(&c, i);
kind = Kind::Str;
} else if ch == '\'' {
let len = if c.get(i + 1) == Some(&'\\') { 4 } else { 3 };
if c.get(i + len - 1) == Some(&'\'') {
i += len;
kind = Kind::Char;
} else {
i += 1;
kind = Kind::Punct;
}
} else if ch.is_ascii_digit() {
while i < c.len() && (c[i].is_ascii_digit() || c[i] == '_') {
i += 1;
}
if c.get(i) == Some(&'.') && c.get(i + 1).map_or(false, |d| d.is_ascii_digit()) {
i += 1;
while i < c.len() && (c[i].is_ascii_digit() || c[i] == '_') {
i += 1;
}
}
kind = Kind::Num;
} else if ch.is_alphabetic() || ch == '_' {
while i < c.len() && (c[i].is_alphanumeric() || c[i] == '_') {
i += 1;
}
kind = Kind::Word;
} else {
let two: String = c[i..(i + 2).min(c.len())].iter().collect();
if ["==", "!=", "<=", ">=", "&&", "||", "+=", "-=", "*=", "/=", "%=", "=>", ".."].contains(&two.as_str()) {
i += 2;
} else {
i += 1;
}
kind = Kind::Punct;
}
let mut text = take(start, i);
if kind == Kind::LineComment {
text = text.trim_end().to_string();
}
out.push(Piece { kind, text, newlines });
newlines = 0;
}
out
}
// ---------- printing ----------
fn is_keyword(p: &Piece) -> bool {
p.kind == Kind::Word && KEYWORDS.contains(&p.text.as_str())
}
fn is_closer(p: &Piece) -> bool {
p.kind == Kind::Punct && matches!(p.text.as_str(), "}" | ")" | "]")
}
fn is_binary(p: &Piece) -> bool {
p.kind == Kind::Punct && BINARY.contains(&p.text.as_str())
}
/// Does a value end with this piece? (Then a following `-` is a subtraction.)
fn ends_value(p: &Piece) -> bool {
match p.kind {
Kind::Num | Kind::Str | Kind::Char => true,
Kind::Word => !is_keyword(p) || matches!(p.text.as_str(), "true" | "false" | "nil"),
Kind::Punct => matches!(p.text.as_str(), ")" | "]" | "}" | "?"),
_ => false,
}
}
/// For each `{`: does it open a block (true) or a struct / map literal (false)?
fn classify_braces(p: &[Piece]) -> Vec<bool> {
let mut out = vec![true; p.len()];
// the groups open at this point; `true` for a block `{`
let mut open: Vec<bool> = Vec::new();
for i in 0..p.len() {
let is_punct = p[i].kind == Kind::Punct;
if is_punct && matches!(p[i].text.as_str(), ")" | "]" | "}") {
open.pop();
}
if is_punct && matches!(p[i].text.as_str(), "(" | "[") {
open.push(false);
}
if !(is_punct && p[i].text == "{") {
continue;
}
// a comma ends an argument or an entry, unless it is in a block (`for k, v in m {`)
let comma_stops = open.last().map_or(false, |b| !*b);
out[i] = 'decide: {
if i == 0 || p[i].newlines > 0 || p[i - 1].text == "=>" {
break 'decide true;
}
// look back over this line, at the same nesting level, for a word that starts a block
let mut depth = 0i32;
let mut j = i;
while j > 0 {
j -= 1;
let t = &p[j];
if t.kind == Kind::Punct {
match t.text.as_str() {
")" | "]" | "}" => depth += 1,
"(" | "[" | "{" => {
depth -= 1;
if depth < 0 {
break;
}
}
"," if depth == 0 && comma_stops => break,
_ => {}
}
} else if depth == 0 && t.kind == Kind::Word && BLOCK_WORDS.contains(&t.text.as_str()) {
break 'decide true;
}
if t.newlines > 0 {
break; // start of the line
}
}
false
};
open.push(out[i]);
}
out
}
struct Open {
line: usize,
/// For `{`: true if it opens a block.
block: bool,
}
fn print(p: &[Piece]) -> String {
let braces = classify_braces(p);
let mut lines: Vec<String> = Vec::new();
let mut cur = String::new();
let mut stack: Vec<Open> = Vec::new();
let mut line_no = 0usize;
let mut cont = false; // the previous line ended with an operator
let mut unary_before = false; // the last piece was a prefix `-` / `!`
// lines that end with a `//` comment, and the byte where the comment starts
let mut comment_lines: std::collections::HashMap<usize, usize> = std::collections::HashMap::new();
let distinct = |s: &[Open]| -> usize {
let mut n = 0;
let mut last = usize::MAX;
for o in s {
if o.line != last {
n += 1;
last = o.line;
}
}
n
};
for (i, t) in p.iter().enumerate() {
let prev = if i > 0 { Some(&p[i - 1]) } else { None };
let line_start = i == 0 || t.newlines > 0;
if line_start {
if i > 0 {
lines.push(std::mem::take(&mut cur));
line_no += 1;
cont = prev.map_or(false, |l| is_binary(l) && l.kind == Kind::Punct) && !unary_before;
}
// blank lines: at most one, none after an opening bracket or before a closing one
let after_open = prev.map_or(true, |l| l.kind == Kind::Punct && matches!(l.text.as_str(), "{" | "(" | "["));
if t.newlines > 1 && !after_open && !is_closer(t) && i > 0 {
lines.push(String::new());
line_no += 1;
}
// indentation: leading closers don't count the groups they close
let mut k = 0;
while i + k < p.len() && is_closer(&p[i + k]) && (k == 0 || p[i + k].newlines == 0) {
k += 1;
}
let remaining = stack.len().saturating_sub(k);
let mut level = distinct(&stack[..remaining]);
if cont && !is_closer(t) {
level += 1;
}
for _ in 0..level {
cur.push_str(INDENT);
}
} else if let Some(pr) = prev {
let mut space = gap(pr, t, if i > 1 { Some(&p[i - 2]) } else { None }, unary_before, &stack, braces[i]);
if t.kind == Kind::Punct && t.text == "}" {
space = match stack.last() {
Some(o) if o.block => !(pr.kind == Kind::Punct && pr.text == "{"),
_ => false,
};
}
if space {
cur.push(' ');
}
}
// is this `-` / `!` a prefix operator?
let is_prefix = t.kind == Kind::Punct
&& (t.text == "!" || (t.text == "-" && (line_start || prev.map_or(true, |l| !ends_value(l)))));
unary_before = is_prefix;
if t.kind == Kind::LineComment && !line_start {
comment_lines.insert(lines.len(), cur.len());
}
cur.push_str(&t.text);
if t.kind == Kind::Punct {
match t.text.as_str() {
"(" | "[" => stack.push(Open { line: line_no, block: false }),
"{" => stack.push(Open { line: line_no, block: braces[i] }),
")" | "]" | "}" => {
stack.pop();
}
_ => {}
}
}
}
lines.push(cur);
while lines.last().map_or(false, |l| l.is_empty()) {
lines.pop();
}
align_comments(&mut lines, &comment_lines);
let mut out = lines.join("\n");
out.push('\n');
out
}
/// Lines in a row that end with a `//` comment get their comments in one column.
/// `comment_lines` holds, for each line that has one, where its code ends.
fn align_comments(lines: &mut [String], comment_lines: &std::collections::HashMap<usize, usize>) {
let mut i = 0;
while i < lines.len() {
if !comment_lines.contains_key(&i) {
i += 1;
continue;
}
let mut j = i;
while j < lines.len() && comment_lines.contains_key(&j) {
j += 1;
}
let width = (i..j).map(|k| lines[k][..comment_lines[&k]].trim_end().chars().count()).max().unwrap_or(0);
for k in i..j {
let at = comment_lines[&k];
let code = lines[k][..at].trim_end().to_string();
let comment = lines[k][at..].trim_start().to_string();
let pad = width - code.chars().count() + 1;
lines[k] = format!("{code}{}{comment}", " ".repeat(pad));
}
i = j;
}
}
/// Is there a space between `prev` and `cur` (which are on the same line)?
fn gap(prev: &Piece, cur: &Piece, before_prev: Option<&Piece>, prev_is_prefix: bool, stack: &[Open], cur_block: bool) -> bool {
let (pt, ct) = (prev.text.as_str(), cur.text.as_str());
let punct = |p: &Piece, s: &str| p.kind == Kind::Punct && p.text == s;
if cur.kind == Kind::LineComment || cur.kind == Kind::BlockComment || prev.kind == Kind::BlockComment {
return true;
}
if prev_is_prefix {
return false;
}
// `import ..a`, `import ...a`: the dots start the path
if prev.kind == Kind::Word && pt == "import" && (ct == "." || ct == "..") {
return true;
}
if prev.kind == Kind::Punct {
match pt {
"(" | "[" | "." | ".." => return false,
"{" => {
// inside a block: `{ x }`; inside a literal: `{a: 1}`
return stack.last().map_or(true, |o| o.block) && !punct(cur, "}");
}
"," | ";" | ":" => return true,
_ => {}
}
}
if cur.kind == Kind::Punct {
match ct {
")" | "]" | "," | ";" | "?" | ":" | "." | ".." => return false,
"{" => {
if cur_block {
return true;
}
// a literal: `Point{...}`, `map[string, int]{}`, but `= {...}`
let attached = (prev.kind == Kind::Word && prev.text.chars().next().map_or(false, |c| c.is_uppercase()))
|| punct(prev, "]");
return !attached;
}
"(" => {
return match prev.kind {
// `step` is only a keyword inside `for ... step n`; before `(` it is a name
Kind::Word => is_keyword(prev) && prev.text != "func" && prev.text != "step",
Kind::Punct => !matches!(pt, ")" | "]" | "}"),
_ => true,
};
}
"[" => {
return match prev.kind {
Kind::Word => is_keyword(prev) && prev.text != "step",
Kind::Punct => !matches!(pt, ")" | "]"),
_ => true,
};
}
_ => {}
}
}
// `[]int`, `[][]int`: no space after an empty `[]`
if punct(prev, "]") && before_prev.map_or(false, |b| punct(b, "[")) && (cur.kind == Kind::Word || punct(cur, "[")) {
return false;
}
true
}