Dino compiler: language, formatter, language server, docs, editor support
Compiler in Rust (lexer, parser, checker, C codegen), maps, chars, dino fmt, dino lsp (diagnostics, formatting, outline, definition, hover, completion, signature help), docs, examples, VS Code/Vim files. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3e2779f447
commit
eaf9567a36
117 files changed
+19970
No files matched your search
+457
@@ -0,0 +1,457 @@
|
||||
//! The code formatter behind `dino fmt`.
|
||||
//!
|
||||
//! It works on the text, not on the syntax tree, so comments and blank lines survive. It
|
||||
//! never moves a line break: it fixes the indentation, the spaces between tokens and runs of
|
||||
//! blank lines. Before handing back the result it lexes both versions and checks that they
|
||||
//! hold exactly the same tokens, so a formatter bug can never change a program.
|
||||
|
||||
use crate::error::Diag;
|
||||
use crate::lexer::{self, StrPart, Tok};
|
||||
|
||||
const INDENT: &str = " ";
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Debug)]
|
||||
enum Kind {
|
||||
Word,
|
||||
Num,
|
||||
Str,
|
||||
Char,
|
||||
Punct,
|
||||
LineComment,
|
||||
BlockComment,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct Piece {
|
||||
kind: Kind,
|
||||
text: String,
|
||||
/// How many line breaks come before this piece (0: same line).
|
||||
newlines: usize,
|
||||
}
|
||||
|
||||
const KEYWORDS: &[&str] = &[
|
||||
"import", "struct", "enum", "interface", "func", "var", "const", "if", "else", "for", "in", "match", "return",
|
||||
"break", "continue", "pub", "shared", "weak", "step",
|
||||
];
|
||||
|
||||
/// Words that make a following `{` open a block (rather than a struct or map literal).
|
||||
const BLOCK_WORDS: &[&str] = &["func", "if", "for", "match", "struct", "enum", "interface", "else"];
|
||||
|
||||
const BINARY: &[&str] = &[
|
||||
"+", "-", "*", "/", "%", "==", "!=", "<", ">", "<=", ">=", "&&", "||", "=", "+=", "-=", "*=", "/=", "%=", "=>",
|
||||
];
|
||||
|
||||
/// Format a whole file. Fails (with the problem) if the file doesn't parse.
|
||||
pub fn format(src: &str) -> Result<String, Diag> {
|
||||
crate::parser::parse(src)?;
|
||||
let pieces = scan(src);
|
||||
let out = print(&pieces);
|
||||
// the safety net: the same tokens in, the same tokens out
|
||||
let before = fingerprint(src)?;
|
||||
let after = fingerprint(&out).map_err(|d| {
|
||||
Diag::new(format!("the formatter produced code that doesn't lex ({}); this is a bug in `dino fmt`", d.msg), d.span)
|
||||
})?;
|
||||
if before != after {
|
||||
return Err(Diag::new(
|
||||
"the formatter would change the meaning of this file; this is a bug in `dino fmt`, nothing was written",
|
||||
crate::ast::Span::default(),
|
||||
));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// The tokens of `src`, without positions and without repeated line ends.
|
||||
fn fingerprint(src: &str) -> Result<Vec<String>, Diag> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for t in lexer::lex(src)? {
|
||||
let s = match &t.tok {
|
||||
Tok::Str(parts) => {
|
||||
let mut s = String::from("str:");
|
||||
for p in parts {
|
||||
match p {
|
||||
StrPart::Text(t) => s.push_str(&format!("T{t:?}")),
|
||||
StrPart::Code { src, .. } => s.push_str(&format!("C{src:?}")),
|
||||
}
|
||||
}
|
||||
s
|
||||
}
|
||||
Tok::Newline if out.last().map_or(true, |l| l == "Newline") => continue,
|
||||
other => format!("{other:?}"),
|
||||
};
|
||||
out.push(s);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// ---------- reading the text ----------
|
||||
|
||||
fn string_end(c: &[char], mut i: usize) -> usize {
|
||||
i += 1; // opening quote
|
||||
loop {
|
||||
match c.get(i) {
|
||||
None => return i,
|
||||
Some('\\') => i += 2,
|
||||
Some('"') => return i + 1,
|
||||
Some('{') => {
|
||||
let mut depth = 1;
|
||||
i += 1;
|
||||
while depth > 0 {
|
||||
match c.get(i) {
|
||||
None => return i,
|
||||
Some('"') => i = string_end(c, i),
|
||||
Some('{') => {
|
||||
depth += 1;
|
||||
i += 1;
|
||||
}
|
||||
Some('}') => {
|
||||
depth -= 1;
|
||||
i += 1;
|
||||
}
|
||||
_ => i += 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => i += 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn scan(src: &str) -> Vec<Piece> {
|
||||
let c: Vec<char> = src.chars().collect();
|
||||
let mut out: Vec<Piece> = Vec::new();
|
||||
let mut i = 0;
|
||||
let mut newlines = 0;
|
||||
let take = |from: usize, to: usize| -> String { c[from..to.min(c.len())].iter().collect() };
|
||||
while i < c.len() {
|
||||
let ch = c[i];
|
||||
if ch == '\n' {
|
||||
newlines += 1;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if ch.is_whitespace() {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let start = i;
|
||||
let kind;
|
||||
if ch == '/' && c.get(i + 1) == Some(&'/') {
|
||||
while i < c.len() && c[i] != '\n' {
|
||||
i += 1;
|
||||
}
|
||||
kind = Kind::LineComment;
|
||||
} else if ch == '/' && c.get(i + 1) == Some(&'*') {
|
||||
i += 2;
|
||||
while i < c.len() && !(c[i] == '*' && c.get(i + 1) == Some(&'/')) {
|
||||
i += 1;
|
||||
}
|
||||
i = (i + 2).min(c.len());
|
||||
kind = Kind::BlockComment;
|
||||
} else if ch == '"' {
|
||||
i = string_end(&c, i);
|
||||
kind = Kind::Str;
|
||||
} else if ch == '\'' {
|
||||
let len = if c.get(i + 1) == Some(&'\\') { 4 } else { 3 };
|
||||
if c.get(i + len - 1) == Some(&'\'') {
|
||||
i += len;
|
||||
kind = Kind::Char;
|
||||
} else {
|
||||
i += 1;
|
||||
kind = Kind::Punct;
|
||||
}
|
||||
} else if ch.is_ascii_digit() {
|
||||
while i < c.len() && (c[i].is_ascii_digit() || c[i] == '_') {
|
||||
i += 1;
|
||||
}
|
||||
if c.get(i) == Some(&'.') && c.get(i + 1).map_or(false, |d| d.is_ascii_digit()) {
|
||||
i += 1;
|
||||
while i < c.len() && (c[i].is_ascii_digit() || c[i] == '_') {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
kind = Kind::Num;
|
||||
} else if ch.is_alphabetic() || ch == '_' {
|
||||
while i < c.len() && (c[i].is_alphanumeric() || c[i] == '_') {
|
||||
i += 1;
|
||||
}
|
||||
kind = Kind::Word;
|
||||
} else {
|
||||
let two: String = c[i..(i + 2).min(c.len())].iter().collect();
|
||||
if ["==", "!=", "<=", ">=", "&&", "||", "+=", "-=", "*=", "/=", "%=", "=>", ".."].contains(&two.as_str()) {
|
||||
i += 2;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
kind = Kind::Punct;
|
||||
}
|
||||
let mut text = take(start, i);
|
||||
if kind == Kind::LineComment {
|
||||
text = text.trim_end().to_string();
|
||||
}
|
||||
out.push(Piece { kind, text, newlines });
|
||||
newlines = 0;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ---------- printing ----------
|
||||
|
||||
fn is_keyword(p: &Piece) -> bool {
|
||||
p.kind == Kind::Word && KEYWORDS.contains(&p.text.as_str())
|
||||
}
|
||||
|
||||
fn is_closer(p: &Piece) -> bool {
|
||||
p.kind == Kind::Punct && matches!(p.text.as_str(), "}" | ")" | "]")
|
||||
}
|
||||
|
||||
fn is_binary(p: &Piece) -> bool {
|
||||
p.kind == Kind::Punct && BINARY.contains(&p.text.as_str())
|
||||
}
|
||||
|
||||
/// Does a value end with this piece? (Then a following `-` is a subtraction.)
|
||||
fn ends_value(p: &Piece) -> bool {
|
||||
match p.kind {
|
||||
Kind::Num | Kind::Str | Kind::Char => true,
|
||||
Kind::Word => !is_keyword(p) || matches!(p.text.as_str(), "true" | "false" | "nil"),
|
||||
Kind::Punct => matches!(p.text.as_str(), ")" | "]" | "}" | "?"),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// For each `{`: does it open a block (true) or a struct / map literal (false)?
|
||||
fn classify_braces(p: &[Piece]) -> Vec<bool> {
|
||||
let mut out = vec![true; p.len()];
|
||||
// the groups open at this point; `true` for a block `{`
|
||||
let mut open: Vec<bool> = Vec::new();
|
||||
for i in 0..p.len() {
|
||||
let is_punct = p[i].kind == Kind::Punct;
|
||||
if is_punct && matches!(p[i].text.as_str(), ")" | "]" | "}") {
|
||||
open.pop();
|
||||
}
|
||||
if is_punct && matches!(p[i].text.as_str(), "(" | "[") {
|
||||
open.push(false);
|
||||
}
|
||||
if !(is_punct && p[i].text == "{") {
|
||||
continue;
|
||||
}
|
||||
// a comma ends an argument or an entry, unless it is in a block (`for k, v in m {`)
|
||||
let comma_stops = open.last().map_or(false, |b| !*b);
|
||||
out[i] = 'decide: {
|
||||
if i == 0 || p[i].newlines > 0 || p[i - 1].text == "=>" {
|
||||
break 'decide true;
|
||||
}
|
||||
// look back over this line, at the same nesting level, for a word that starts a block
|
||||
let mut depth = 0i32;
|
||||
let mut j = i;
|
||||
while j > 0 {
|
||||
j -= 1;
|
||||
let t = &p[j];
|
||||
if t.kind == Kind::Punct {
|
||||
match t.text.as_str() {
|
||||
")" | "]" | "}" => depth += 1,
|
||||
"(" | "[" | "{" => {
|
||||
depth -= 1;
|
||||
if depth < 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
"," if depth == 0 && comma_stops => break,
|
||||
_ => {}
|
||||
}
|
||||
} else if depth == 0 && t.kind == Kind::Word && BLOCK_WORDS.contains(&t.text.as_str()) {
|
||||
break 'decide true;
|
||||
}
|
||||
if t.newlines > 0 {
|
||||
break; // start of the line
|
||||
}
|
||||
}
|
||||
false
|
||||
};
|
||||
open.push(out[i]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
struct Open {
|
||||
line: usize,
|
||||
/// For `{`: true if it opens a block.
|
||||
block: bool,
|
||||
}
|
||||
|
||||
fn print(p: &[Piece]) -> String {
|
||||
let braces = classify_braces(p);
|
||||
let mut lines: Vec<String> = Vec::new();
|
||||
let mut cur = String::new();
|
||||
let mut stack: Vec<Open> = Vec::new();
|
||||
let mut line_no = 0usize;
|
||||
let mut cont = false; // the previous line ended with an operator
|
||||
let mut unary_before = false; // the last piece was a prefix `-` / `!`
|
||||
// lines that end with a `//` comment, and the byte where the comment starts
|
||||
let mut comment_lines: std::collections::HashMap<usize, usize> = std::collections::HashMap::new();
|
||||
|
||||
let distinct = |s: &[Open]| -> usize {
|
||||
let mut n = 0;
|
||||
let mut last = usize::MAX;
|
||||
for o in s {
|
||||
if o.line != last {
|
||||
n += 1;
|
||||
last = o.line;
|
||||
}
|
||||
}
|
||||
n
|
||||
};
|
||||
|
||||
for (i, t) in p.iter().enumerate() {
|
||||
let prev = if i > 0 { Some(&p[i - 1]) } else { None };
|
||||
let line_start = i == 0 || t.newlines > 0;
|
||||
if line_start {
|
||||
if i > 0 {
|
||||
lines.push(std::mem::take(&mut cur));
|
||||
line_no += 1;
|
||||
cont = prev.map_or(false, |l| is_binary(l) && l.kind == Kind::Punct) && !unary_before;
|
||||
}
|
||||
// blank lines: at most one, none after an opening bracket or before a closing one
|
||||
let after_open = prev.map_or(true, |l| l.kind == Kind::Punct && matches!(l.text.as_str(), "{" | "(" | "["));
|
||||
if t.newlines > 1 && !after_open && !is_closer(t) && i > 0 {
|
||||
lines.push(String::new());
|
||||
line_no += 1;
|
||||
}
|
||||
// indentation: leading closers don't count the groups they close
|
||||
let mut k = 0;
|
||||
while i + k < p.len() && is_closer(&p[i + k]) && (k == 0 || p[i + k].newlines == 0) {
|
||||
k += 1;
|
||||
}
|
||||
let remaining = stack.len().saturating_sub(k);
|
||||
let mut level = distinct(&stack[..remaining]);
|
||||
if cont && !is_closer(t) {
|
||||
level += 1;
|
||||
}
|
||||
for _ in 0..level {
|
||||
cur.push_str(INDENT);
|
||||
}
|
||||
} else if let Some(pr) = prev {
|
||||
let mut space = gap(pr, t, if i > 1 { Some(&p[i - 2]) } else { None }, unary_before, &stack, braces[i]);
|
||||
if t.kind == Kind::Punct && t.text == "}" {
|
||||
space = match stack.last() {
|
||||
Some(o) if o.block => !(pr.kind == Kind::Punct && pr.text == "{"),
|
||||
_ => false,
|
||||
};
|
||||
}
|
||||
if space {
|
||||
cur.push(' ');
|
||||
}
|
||||
}
|
||||
|
||||
// is this `-` / `!` a prefix operator?
|
||||
let is_prefix = t.kind == Kind::Punct
|
||||
&& (t.text == "!" || (t.text == "-" && (line_start || prev.map_or(true, |l| !ends_value(l)))));
|
||||
unary_before = is_prefix;
|
||||
|
||||
if t.kind == Kind::LineComment && !line_start {
|
||||
comment_lines.insert(lines.len(), cur.len());
|
||||
}
|
||||
cur.push_str(&t.text);
|
||||
|
||||
if t.kind == Kind::Punct {
|
||||
match t.text.as_str() {
|
||||
"(" | "[" => stack.push(Open { line: line_no, block: false }),
|
||||
"{" => stack.push(Open { line: line_no, block: braces[i] }),
|
||||
")" | "]" | "}" => {
|
||||
stack.pop();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
lines.push(cur);
|
||||
while lines.last().map_or(false, |l| l.is_empty()) {
|
||||
lines.pop();
|
||||
}
|
||||
align_comments(&mut lines, &comment_lines);
|
||||
let mut out = lines.join("\n");
|
||||
out.push('\n');
|
||||
out
|
||||
}
|
||||
|
||||
/// Lines in a row that end with a `//` comment get their comments in one column.
|
||||
/// `comment_lines` holds, for each line that has one, where its code ends.
|
||||
fn align_comments(lines: &mut [String], comment_lines: &std::collections::HashMap<usize, usize>) {
|
||||
let mut i = 0;
|
||||
while i < lines.len() {
|
||||
if !comment_lines.contains_key(&i) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let mut j = i;
|
||||
while j < lines.len() && comment_lines.contains_key(&j) {
|
||||
j += 1;
|
||||
}
|
||||
let width = (i..j).map(|k| lines[k][..comment_lines[&k]].trim_end().chars().count()).max().unwrap_or(0);
|
||||
for k in i..j {
|
||||
let at = comment_lines[&k];
|
||||
let code = lines[k][..at].trim_end().to_string();
|
||||
let comment = lines[k][at..].trim_start().to_string();
|
||||
let pad = width - code.chars().count() + 1;
|
||||
lines[k] = format!("{code}{}{comment}", " ".repeat(pad));
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
}
|
||||
|
||||
/// Is there a space between `prev` and `cur` (which are on the same line)?
|
||||
fn gap(prev: &Piece, cur: &Piece, before_prev: Option<&Piece>, prev_is_prefix: bool, stack: &[Open], cur_block: bool) -> bool {
|
||||
let (pt, ct) = (prev.text.as_str(), cur.text.as_str());
|
||||
let punct = |p: &Piece, s: &str| p.kind == Kind::Punct && p.text == s;
|
||||
|
||||
if cur.kind == Kind::LineComment || cur.kind == Kind::BlockComment || prev.kind == Kind::BlockComment {
|
||||
return true;
|
||||
}
|
||||
if prev_is_prefix {
|
||||
return false;
|
||||
}
|
||||
if prev.kind == Kind::Punct {
|
||||
match pt {
|
||||
"(" | "[" | "." | ".." => return false,
|
||||
"{" => {
|
||||
// inside a block: `{ x }`; inside a literal: `{a: 1}`
|
||||
return stack.last().map_or(true, |o| o.block) && !punct(cur, "}");
|
||||
}
|
||||
"," | ";" | ":" => return true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if cur.kind == Kind::Punct {
|
||||
match ct {
|
||||
")" | "]" | "," | ";" | "?" | ":" | "." | ".." => return false,
|
||||
"{" => {
|
||||
if cur_block {
|
||||
return true;
|
||||
}
|
||||
// a literal: `Point{...}`, `map[string, int]{}`, but `= {...}`
|
||||
let attached = (prev.kind == Kind::Word && prev.text.chars().next().map_or(false, |c| c.is_uppercase()))
|
||||
|| punct(prev, "]");
|
||||
return !attached;
|
||||
}
|
||||
"(" => {
|
||||
return match prev.kind {
|
||||
Kind::Word => is_keyword(prev) && prev.text != "func",
|
||||
Kind::Punct => !matches!(pt, ")" | "]" | "}"),
|
||||
_ => true,
|
||||
};
|
||||
}
|
||||
"[" => {
|
||||
return match prev.kind {
|
||||
Kind::Word => is_keyword(prev),
|
||||
Kind::Punct => !matches!(pt, ")" | "]"),
|
||||
_ => true,
|
||||
};
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
// `[]int`, `[][]int`: no space after an empty `[]`
|
||||
if punct(prev, "]") && before_prev.map_or(false, |b| punct(b, "[")) && (cur.kind == Kind::Word || punct(cur, "[")) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
Reference in new issue
Block a user