Files
ink-note-app/electron/vault.cjs
T
Your NameandClaude Sonnet 5.5 d5d25eaf46 perf: faster startup and long pages
- open the window while the notes folder loads; rebuild the search index after the page list is sent
- load mermaid and katex on demand
- vault.list reads files in parallel batches
- resolveLink uses a per-page-list lookup table
- very long pages (>300 blocks) skip layout of off-screen blocks
- perf-large: add a long-page case, fix the tasks poll

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-15 12:25:45 +02:00

547 lines
24 KiB
JavaScript

'use strict';
// The vault is a plain folder of Markdown files.
// "Projects.md" -> page "Projects"
// "Projects/Note app roadmap.md" -> child page of "Projects"
// A folder without a matching .md file still shows up as an (empty) page.
// Page ids are paths relative to the vault, without ".md", always with "/".
// Canvases ("Ideas/Map.canvas", see canvas.cjs) are pages too; their id keeps the ".canvas", as links to them do.
const fs = require('node:fs');
const fsp = fs.promises;
const path = require('node:path');
const { parseFrontMatter, serializeFrontMatter } = require('./frontmatter.cjs');
const { isCanvasId, canvasText, EMPTY_CANVAS } = require('./canvas.cjs');
const ILLEGAL = /[\\/:*?"<>|\u0000-\u001f]/g;
const SKIP_DIRS = new Set(['node_modules']);
function cleanTitle(t) {
const s = String(t || '').replace(ILLEGAL, '-').replace(/\s+/g, ' ').trim().replace(/^\.+/, '').slice(0, 120).trim();
return s || 'Untitled';
}
const today = (d = new Date()) => {
const z = (n) => String(n).padStart(2, '0');
return `${d.getFullYear()}-${z(d.getMonth() + 1)}-${z(d.getDate())}`;
};
const parentOf = (id) => (id.includes('/') ? id.slice(0, id.lastIndexOf('/')) : null);
const PAGE_KEYS = new Set(['icon', 'cover', 'tags', 'created']);
const titleOf = (id) => id.slice(id.lastIndexOf('/') + 1).replace(/\.canvas$/i, '');
const extOf = (id) => (isCanvasId(id) ? '.canvas' : '');
const noCanvasParent = (parent) => { if (parent && isCanvasId(parent)) throw new Error('A canvas cannot hold pages.'); };
const asArray = (v) => (Array.isArray(v) ? v : typeof v === 'string' && v ? [v] : []).map(String);
async function exists(p) {
try { await fsp.access(p); return true; } catch { return false; }
}
// Version history limits (Settings): at most one version per `gap` of editing, the newest `keep` per page.
const limits = { gap: 5 * 60 * 1000, keep: 100 };
function setLimits({ historyKeep, historyGapMin } = {}) {
if (Number.isFinite(historyKeep) && historyKeep >= 1) limits.keep = Math.floor(historyKeep);
if (Number.isFinite(historyGapMin) && historyGapMin >= 1) limits.gap = Math.floor(historyGapMin) * 60 * 1000;
}
const isLocalPath = (src) => !!src && !/^[a-z][a-z0-9+.-]*:/i.test(src) && !src.startsWith('/') && !src.startsWith('\\');
const cleanFileName = (name) => {
const ext = (path.extname(String(name)).toLowerCase().match(/^\.[a-z0-9]{1,5}$/) || ['.png'])[0];
const base = path.basename(String(name), path.extname(String(name))).replace(ILLEGAL, '-').replace(/\s+/g, ' ').trim().replace(/^\.+/, '').slice(0, 80);
return (base || 'image') + ext;
};
async function freeFile(dir, name) {
const ext = path.extname(name), base = name.slice(0, name.length - ext.length);
for (let n = 1; n < 1000; n++) {
const f = n === 1 ? name : `${base} ${n}${ext}`;
if (!(await exists(path.join(dir, f)))) return f;
}
throw new Error('Could not find a free file name.');
}
class Vault {
constructor(root, { trash } = {}) {
this.root = path.resolve(root);
this.trash = trash;
this.recent = new Map(); // id -> time of our own write, to ignore watcher echoes
this.cache = new Map(); // fullPath -> { mtime, page }
}
#inside(p) {
if (p !== this.root && !p.startsWith(this.root + path.sep)) throw new Error('That path is outside the notes folder.');
return p;
}
abs(id) { return this.#inside(path.resolve(this.root, isCanvasId(id) ? id : id + '.md')); }
dir(id) { return this.#inside(path.resolve(this.root, id || '.')); }
#touch(id) { this.recent.set(id, Date.now()); }
isRecent(id) { const t = this.recent.get(id); return t !== undefined && Date.now() - t < 1500; }
async list() {
// Each folder yields its pages (files first, then its subfolders, in directory order). Files are read in
// parallel batches and subfolders are walked together; the order of the result stays the same.
const readPage = async (dir, parent, e) => {
const canvas = /\.canvas$/i.test(e.name);
const title = canvas ? e.name.slice(0, -7) : e.name.slice(0, -3);
const id = parent ? `${parent}/${canvas ? e.name : title}` : canvas ? e.name : title;
const full = path.join(dir, e.name);
try {
const st = await fsp.stat(full);
const cached = this.cache.get(full);
let page;
if (cached && cached.mtime === st.mtimeMs) {
page = cached.page;
// update ID and parent in case it moved/renamed but mtime stayed the same
page.id = id;
page.parent = parent || null;
page.title = title;
} else {
const raw = await fsp.readFile(full, 'utf8');
// A canvas's "body" is its text and the notes on it as [[links]], for search, backlinks and the graph.
const { meta, body } = canvas ? { meta: {}, body: canvasText(raw) } : parseFrontMatter(raw);
// Front matter beyond the page's own keys: a database record's property values (see src/lib/database.js).
const props = {};
for (const k of Object.keys(meta)) if (!PAGE_KEYS.has(k)) props[k] = meta[k];
page = {
...(canvas ? { kind: 'canvas' } : {}),
props,
...(meta['ink-database'] ? { db: true } : {}),
id, parent: parent || null, title, exists: true, mtime: st.mtimeMs,
icon: meta.icon || null, cover: meta.cover || null, tags: asArray(meta.tags),
created: meta.created || today(st.birthtime && st.birthtime.getTime() ? st.birthtime : st.mtime),
body
};
this.cache.set(full, { mtime: st.mtimeMs, page });
}
return page;
} catch { return null; /* unreadable file: skip */ }
};
const walk = async (dir, parent) => {
let entries;
try { entries = await fsp.readdir(dir, { withFileTypes: true }); } catch { return []; }
const files = entries.filter((e) => !e.name.startsWith('.') && e.isFile() && /\.(md|canvas)$/i.test(e.name));
const out = [];
const taken = new Set();
for (let i = 0; i < files.length; i += 64) {
const batch = await Promise.all(files.slice(i, i + 64).map((e) => readPage(dir, parent, e)));
for (const page of batch) if (page) { out.push(page); taken.add(page.id); }
}
const dirs = entries.filter((e) => !e.name.startsWith('.') && e.isDirectory() && !SKIP_DIRS.has(e.name));
const subs = await Promise.all(dirs.map((e) => walk(path.join(dir, e.name), parent ? `${parent}/${e.name}` : e.name)));
dirs.forEach((e, i) => {
const id = parent ? `${parent}/${e.name}` : e.name;
if (!taken.has(id)) {
out.push({ id, parent: parent || null, title: e.name, exists: false, mtime: 0, icon: null, cover: null, tags: [], created: null, body: '' });
taken.add(id);
}
for (const p of subs[i]) out.push(p);
});
return out;
};
const pages = new Map();
for (const p of await walk(this.root, '')) pages.set(p.id, p);
const active = new Set([...pages.values()].filter(p => p.exists).map(p => this.abs(p.id)));
for (const k of this.cache.keys()) {
if (!active.has(k)) this.cache.delete(k);
}
return [...pages.values()];
}
async read(id) {
const file = this.abs(id);
try {
const [raw, st] = await Promise.all([fsp.readFile(file, 'utf8'), fsp.stat(file)]);
const { meta, body } = isCanvasId(id) ? { meta: {}, body: raw } : parseFrontMatter(raw);
if (meta.tags !== undefined) meta.tags = asArray(meta.tags);
return { id, title: titleOf(id), meta, body, exists: true, mtime: st.mtimeMs };
} catch (err) {
if (err.code !== 'ENOENT') throw err;
return { id, title: titleOf(id), meta: {}, body: '', exists: false, mtime: 0 };
}
}
async write(id, meta, body) {
const file = this.abs(id);
await fsp.mkdir(path.dirname(file), { recursive: true });
const text = isCanvasId(id) ? String(body || '') : serializeFrontMatter(meta) + String(body || '');
const tmp = path.join(path.dirname(file), `.${path.basename(file)}.${process.pid}.tmp`);
this.#touch(id);
await fsp.writeFile(tmp, text, 'utf8');
await fsp.rename(tmp, file);
this.#touch(id);
await this.snapshot(id, text);
return (await fsp.stat(file)).mtimeMs;
}
/* version history
.notes/history/<page id>/<time>.md holds copies of a page as it was. Saves happen every few
hundred ms while typing, so a copy is kept at most every limits.gap (and never twice the same),
the newest limits.keep per page. */
get historyRoot() { return path.join(this.root, '.notes', 'history'); }
#historyDir(id) {
const dir = path.resolve(this.historyRoot, id);
if (!dir.startsWith(this.historyRoot + path.sep)) throw new Error('That path is outside the notes folder.');
return dir;
}
async #versions(id) {
try {
return (await fsp.readdir(this.#historyDir(id))).filter((v) => /^\d+\.md$/.test(v)).map((v) => parseInt(v, 10)).sort((a, b) => b - a);
} catch { return []; }
}
// Keep `text` (the page's full file) as a version. force: skip the time gap (used before a restore).
async snapshot(id, text, { force = false } = {}) {
try {
const dir = this.#historyDir(id);
const [newest] = await this.#versions(id);
if (newest && !force) {
if (Date.now() - newest < limits.gap) return;
}
if (newest && (await fsp.readFile(path.join(dir, `${newest}.md`), 'utf8').catch(() => null)) === text) return;
await fsp.mkdir(dir, { recursive: true });
await fsp.writeFile(path.join(dir, `${Math.max(Date.now(), (newest || 0) + 1)}.md`), text, 'utf8');
const all = await this.#versions(id);
for (const v of all.slice(limits.keep)) await fsp.unlink(path.join(dir, `${v}.md`)).catch(() => {});
} catch (err) {
console.error('Could not save a version', err);
}
}
// Keep the file as it is on disk right now as a version (before something else overwrites it).
async keepCurrentVersion(id) {
const text = await fsp.readFile(this.abs(id), 'utf8');
await this.snapshot(id, text, { force: true });
}
listHistory(id) { return this.#versions(id); }
async readHistory(id, ts) {
if (!/^\d+$/.test(String(ts))) throw new Error('Unknown version.');
const text = await fsp.readFile(path.join(this.#historyDir(id), `${ts}.md`), 'utf8');
return isCanvasId(id) ? { meta: {}, body: text } : parseFrontMatter(text); // a canvas has no front matter
}
// Put version `ts` back. What's there now is kept as a version first, so a restore can itself be undone.
async restoreHistory(id, ts) {
const { meta, body } = await this.readHistory(id, ts);
try { await this.snapshot(id, await fsp.readFile(this.abs(id), 'utf8'), { force: true }); } catch { /* page file gone */ }
return this.write(id, meta, body);
}
// History follows a page (and its sub-pages) when it's renamed or moved.
async #moveHistory(id, next) {
const from = this.#historyDir(id), to = this.#historyDir(next);
if (!(await exists(from)) || (await exists(to))) return;
await fsp.mkdir(path.dirname(to), { recursive: true });
await fsp.rename(from, to).catch((err) => console.error('Could not move history', err));
}
async #freeName(parent, wanted, except, ext = '') {
const base = cleanTitle(wanted);
for (let n = 1; n < 1000; n++) {
const title = (n === 1 ? base : `${base} ${n}`) + ext;
const id = parent ? `${parent}/${title}` : title;
if (except && id.toLowerCase() === except.toLowerCase()) return id;
if (!(await exists(this.abs(id))) && !(await exists(this.dir(id)))) return id;
}
throw new Error('Could not find a free file name.');
}
async create(parent, title, meta = {}, body = '') {
noCanvasParent(parent);
if (parent) await fsp.mkdir(this.dir(parent), { recursive: true });
const id = await this.#freeName(parent, title || 'Untitled');
await this.write(id, { created: today(), ...meta }, body);
return id;
}
async createCanvas(parent, title) {
noCanvasParent(parent);
if (parent) await fsp.mkdir(this.dir(parent), { recursive: true });
const id = await this.#freeName(parent, title || 'Untitled canvas', undefined, '.canvas');
await this.write(id, {}, EMPTY_CANVAS);
return id;
}
// A copy of a page, board or canvas next to the original ("Name copy", "Name copy 2", …), with its sub-pages and
// pictures (the page's folder) when it has any. Returns the copy's id.
async duplicate(id) {
const src = this.abs(id);
if (!(await exists(src))) throw new Error('That page has no file to copy.');
const next = await this.#freeName(parentOf(id), titleOf(id) + ' copy', undefined, extOf(id));
await fsp.mkdir(path.dirname(this.abs(next)), { recursive: true });
if (!isCanvasId(id) && (await exists(this.dir(id)))) await fsp.cp(this.dir(id), this.dir(next), { recursive: true });
this.#touch(next);
await fsp.copyFile(src, this.abs(next));
this.#touch(next);
await this.snapshot(next, await fsp.readFile(this.abs(next), 'utf8'));
return next;
}
// The folder templates live in (made on demand, so there's something to open).
async ensureTemplatesDir() {
const dir = path.join(this.root, '.notes', 'templates');
await fsp.mkdir(dir, { recursive: true });
return dir;
}
async listTemplates() {
try {
const tplDir = path.join(this.root, '.notes', 'templates');
if (!(await exists(tplDir))) return [];
const files = await fsp.readdir(tplDir);
return files.filter(f => f.endsWith('.md')).map(f => f.slice(0, -3));
} catch {
return [];
}
}
async getTemplate(name) {
try {
const tplFile = path.join(this.root, '.notes', 'templates', name + '.md');
if (!(await exists(tplFile))) return null;
const raw = await fsp.readFile(tplFile, 'utf8');
return parseFrontMatter(raw);
} catch {
return null;
}
}
async rename(id, title) {
const parent = parentOf(id);
if (cleanTitle(title) === titleOf(id)) return id;
const next = await this.#freeName(parent, title, id, extOf(id));
if (next === id) return id;
this.#touch(id); this.#touch(next);
if (await exists(this.abs(id))) await fsp.rename(this.abs(id), this.abs(next));
if (!isCanvasId(id) && (await exists(this.dir(id)))) await fsp.rename(this.dir(id), this.dir(next));
await this.#moveHistory(id, next);
return next;
}
// Move a page (and its sub-pages) under another page, or to the top level when newParent is null.
// Images the page shows from its own folder go with it, so its relative links keep working.
async move(id, newParent) {
newParent = newParent || null;
if (newParent === parentOf(id)) return id;
if (newParent && (newParent === id || newParent.startsWith(id + '/'))) throw new Error('A page cannot go inside itself.');
noCanvasParent(newParent);
if (newParent) await fsp.mkdir(this.dir(newParent), { recursive: true });
const next = await this.#freeName(newParent, titleOf(id), undefined, extOf(id));
const from = path.dirname(this.abs(id)), to = path.dirname(this.abs(next));
this.#touch(id); this.#touch(next);
if (await exists(this.abs(id))) {
await fsp.rename(this.abs(id), this.abs(next));
if (!isCanvasId(next)) await this.#moveImages(next, from, to);
}
if (!isCanvasId(id) && (await exists(this.dir(id)))) await fsp.rename(this.dir(id), this.dir(next));
await this.#moveHistory(id, next);
return next;
}
async #moveImages(id, from, to) {
const { meta, body } = await this.read(id);
let changed = false;
const done = new Map(); // old path -> new name, for images shown more than once
const relocate = async (src) => {
if (!isLocalPath(src)) return src;
const old = path.resolve(from, src);
if (path.dirname(old) !== from) return src;
let name = done.get(old);
if (!name) {
if (!(await exists(old))) return src;
name = await freeFile(to, path.basename(old));
await fsp.rename(old, path.join(to, name));
done.set(old, name);
}
if (name !== path.basename(old)) changed = true;
return name;
};
// a picture used as the page's cover goes with it too
let nextMeta = meta;
if (typeof meta.cover === 'string' && /\.(png|jpe?g|gif|webp|avif|svg|bmp)$/i.test(meta.cover)) {
const name = await relocate(meta.cover);
if (name !== meta.cover) nextMeta = { ...meta, cover: name };
}
const out = [];
for (const line of body.split('\n')) {
let m = line.match(/^(\s*!\[[^\]]*\]\(\s*)(<[^>]+>|[^)\s]+)(.*)$/);
if (m) {
const raw = m[2].replace(/^<|>$/g, '');
let src = raw;
try { src = decodeURI(raw); } catch { /* as written */ }
const name = await relocate(src);
out.push(name === src ? line : m[1] + name.replace(/[ ()<>]/g, encodeURIComponent) + m[3]);
continue;
}
m = line.match(/^(\s*!\[\[)([^\]|]+)(.*)$/);
if (m) {
const name = await relocate(m[2].trim());
out.push(name === m[2].trim() ? line : m[1] + name + m[3]);
continue;
}
out.push(line);
}
if (changed) await this.write(id, nextMeta, out.join('\n'));
}
// Store image bytes next to page `id`; returns the file name to use in ![](name).
async saveImage(id, name, bytes) {
const dir = path.dirname(this.abs(id));
await fsp.mkdir(dir, { recursive: true });
const file = await freeFile(dir, cleanFileName(name));
await fsp.writeFile(path.join(dir, file), bytes);
return file;
}
// Absolute path of an image `src` as written in page `id`, or null if it isn't a file in the vault.
imagePath(id, src) {
if (!isLocalPath(src)) return null;
try { return this.#inside(path.resolve(path.dirname(this.abs(id)), src)); } catch { return null; }
}
/* trash
Deleted pages go to .notes/trash/<entry>/ so the app can list and restore them:
info.json { id, title, icon, pages, deletedAt }
files/ the page's .md and its folder of sub-pages, as they were
Deleting from there (or after 30 days) hands the entry to the system trash. */
get trashRoot() { return path.join(this.root, '.notes', 'trash'); }
// Move a page and its sub-pages to the trash. Returns the trash entry, or null if there was nothing to move.
async remove(id) {
const targets = isCanvasId(id) ? [this.abs(id)] : [this.abs(id), this.dir(id)]; // a canvas has no folder of sub-pages
const found = [];
for (const t of targets) if (await exists(t)) found.push(t);
if (!found.length) return null;
const page = await this.read(id);
const below = (await this.list()).filter((p) => p.exists && p.id.startsWith(id + '/')).length;
const entry = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 7)}`;
const files = path.join(this.trashRoot, entry, 'files');
await fsp.mkdir(files, { recursive: true });
this.#touch(id);
for (const t of found) await fsp.rename(t, path.join(files, path.basename(t)));
const info = { id, title: titleOf(id), icon: page.meta.icon || null, pages: below + (page.exists ? 1 : 0), deletedAt: Date.now() };
await fsp.writeFile(path.join(this.trashRoot, entry, 'info.json'), JSON.stringify(info, null, 2));
return entry;
}
async listTrash() {
let names;
try { names = await fsp.readdir(this.trashRoot); } catch { return []; }
const out = [];
for (const entry of names) {
try {
const info = JSON.parse(await fsp.readFile(path.join(this.trashRoot, entry, 'info.json'), 'utf8'));
out.push({ entry, ...info });
} catch { /* half-written entry: skip */ }
}
return out.sort((a, b) => b.deletedAt - a.deletedAt);
}
#entry(entry) {
if (!/^[a-z0-9-]+$/i.test(String(entry))) throw new Error('Unknown trash item.');
return path.join(this.trashRoot, entry);
}
// Put a trashed page back where it was (with a number added if that name is taken now). Returns its id.
async restore(entry) {
const base = this.#entry(entry);
const info = JSON.parse(await fsp.readFile(path.join(base, 'info.json'), 'utf8'));
const parent = parentOf(info.id);
if (parent) await fsp.mkdir(this.dir(parent), { recursive: true });
const next = await this.#freeName(parent, titleOf(info.id), undefined, extOf(info.id));
const title = titleOf(info.id);
this.#touch(next);
const md = path.join(base, 'files', path.basename(this.abs(info.id))), folder = path.join(base, 'files', title);
if (await exists(md)) await fsp.rename(md, this.abs(next));
if (await exists(folder)) await fsp.rename(folder, this.dir(next));
await fsp.rm(base, { recursive: true, force: true });
return next;
}
// Delete for good: to the system trash when there is one. No entry = everything.
async purge(entry) {
const entries = entry ? [entry] : (await this.listTrash()).map((t) => t.entry);
for (const e of entries) {
const base = this.#entry(e);
if (!(await exists(base))) continue;
if (this.trash) await this.trash(base);
else await fsp.rm(base, { recursive: true, force: true });
}
}
async purgeOlderThan(days) {
const cutoff = Date.now() - days * 864e5;
for (const t of await this.listTrash()) if (t.deletedAt < cutoff) await this.purge(t.entry);
}
// Calls back with the ids of pages changed outside the app.
// Calls onChange(ids) (debounced) when pages change on disk behind the app's back.
// One non-recursive watcher per note folder: Node's recursive watcher on Linux stopped delivering events once
// the app had created folders of its own (.notes/history), and it would also report all the churn inside .notes.
// Dot folders and the app's own are never watched.
watch(onChange) {
const pending = new Set();
const watchers = new Map(); // absolute folder -> FSWatcher
let timer = null;
let resyncTimer = null;
let closed = false;
const flush = () => {
const ids = [...pending].filter((id) => !this.isRecent(id));
pending.clear();
if (ids.length) onChange(ids);
};
const note = (id) => {
pending.add(id);
clearTimeout(timer);
timer = setTimeout(flush, 250);
};
const add = async (dir) => {
if (closed || watchers.has(dir)) return;
let w;
try {
w = fs.watch(dir, (evt, name) => onEvent(dir, evt, name));
} catch {
return;
}
w.on('error', () => { try { w.close(); } catch { /* gone */ } watchers.delete(dir); });
watchers.set(dir, w);
let entries = [];
try { entries = await fsp.readdir(dir, { withFileTypes: true }); } catch { return; }
for (const e of entries) {
if (e.isDirectory() && !e.name.startsWith('.') && !SKIP_DIRS.has(e.name)) await add(path.join(dir, e.name));
}
};
// Folders that appeared, moved or vanished: look at the whole tree again (a watched folder that is renamed
// keeps reporting under its old name).
const resync = () => {
clearTimeout(resyncTimer);
resyncTimer = setTimeout(async () => {
if (closed) return;
for (const w of watchers.values()) { try { w.close(); } catch { /* gone */ } }
watchers.clear();
await add(this.root);
}, 200);
};
const onEvent = (dir, evt, name) => {
if (!name) return;
name = String(name);
if (name.startsWith('.')) return; // temp files, .notes, .obsidian, .git
const full = path.join(dir, name);
const rel = path.relative(this.root, full).split(path.sep).join('/');
if (SKIP_DIRS.has(name)) return;
note(rel.replace(/\.md$/i, ''));
if (evt === 'rename') {
fsp.stat(full).then((st) => { if (st.isDirectory()) resync(); }, () => { if (watchers.has(full) || !/\.[a-z0-9]+$/i.test(name)) resync(); });
}
};
add(this.root);
return () => {
closed = true;
clearTimeout(timer);
clearTimeout(resyncTimer);
for (const w of watchers.values()) { try { w.close(); } catch { /* gone */ } }
watchers.clear();
};
}
}
module.exports = { setLimits, Vault, cleanTitle, today, titleOf, parentOf, isCanvasId };