feat(electron): main process, vault, search index, links, import, capture
Vault on plain .md files with FTS5 index, link rewriting, canvas files, history/trash/backup, link previews, image fetch, tasks and unlinked mentions, Notion/Obsidian importer, quick-capture window and shortcut. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0b5f8bfa8b
commit
313c41f867
16 files changed
+2706
-15
No files matched your search
@@ -0,0 +1,148 @@
|
||||
'use strict';
|
||||
// Bringing notes in from another app: an Obsidian vault or a Notion "Markdown & CSV" export, as a folder or a .zip.
|
||||
// Everything goes into one new top-level folder of the notes, so nothing existing is touched.
|
||||
// - Notion's 32-character ids are taken out of file and folder names, and its relative Markdown links
|
||||
// ([Title](Folder/Page%20abc….md)) become [[wiki links]] to the imported pages.
|
||||
// - Obsidian's .obsidian folder and Notion's _all.csv files are left out; other files (pictures, PDFs) are copied.
|
||||
const fs = require('node:fs');
|
||||
const fsp = fs.promises;
|
||||
const path = require('node:path');
|
||||
const zlib = require('node:zlib');
|
||||
|
||||
const SKIP_DIR = /^(\.obsidian|\.trash|\.git|\.notes|__MACOSX|node_modules)$/;
|
||||
const MAX_FILES = 50000;
|
||||
const MAX_FILE = 200 * 1024 * 1024;
|
||||
|
||||
/* zip reading: central directory, stored and deflated entries */
|
||||
function readZip(file) {
|
||||
const buf = fs.readFileSync(file);
|
||||
let end = -1;
|
||||
for (let i = buf.length - 22; i >= Math.max(0, buf.length - 65557); i--) if (buf.readUInt32LE(i) === 0x06054b50) { end = i; break; }
|
||||
if (end < 0) throw new Error('That file is not a zip archive.');
|
||||
const count = buf.readUInt16LE(end + 10);
|
||||
let at = buf.readUInt32LE(end + 16);
|
||||
const out = [];
|
||||
for (let n = 0; n < count && n < MAX_FILES; n++) {
|
||||
if (buf.readUInt32LE(at) !== 0x02014b50) break;
|
||||
const method = buf.readUInt16LE(at + 10);
|
||||
const csize = buf.readUInt32LE(at + 20);
|
||||
const size = buf.readUInt32LE(at + 24);
|
||||
const nameLen = buf.readUInt16LE(at + 28), extraLen = buf.readUInt16LE(at + 30), commentLen = buf.readUInt16LE(at + 32);
|
||||
const local = buf.readUInt32LE(at + 42);
|
||||
const name = buf.toString('utf8', at + 46, at + 46 + nameLen);
|
||||
at += 46 + nameLen + extraLen + commentLen;
|
||||
if (name.endsWith('/') || size > MAX_FILE) continue;
|
||||
const dataAt = local + 30 + buf.readUInt16LE(local + 26) + buf.readUInt16LE(local + 28);
|
||||
out.push({
|
||||
rel: name,
|
||||
read: () => {
|
||||
const raw = buf.subarray(dataAt, dataAt + csize);
|
||||
return method === 0 ? raw : zlib.inflateRawSync(raw);
|
||||
}
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function* walkDir(root, rel = '') {
|
||||
let entries;
|
||||
try { entries = await fsp.readdir(path.join(root, rel), { withFileTypes: true }); } catch { return; }
|
||||
for (const e of entries) {
|
||||
if (e.isSymbolicLink()) continue;
|
||||
const r = rel ? `${rel}/${e.name}` : e.name;
|
||||
if (e.isDirectory()) { if (!SKIP_DIR.test(e.name)) yield* walkDir(root, r); }
|
||||
else if (e.isFile()) yield r;
|
||||
}
|
||||
}
|
||||
|
||||
// "Page 0a1b2c3d4e5f60718293a4b5c6d7e8f9.md" -> "Page.md"; a plain name is left alone.
|
||||
const NOTION_ID = /\s+[0-9a-f]{32}(?=\.[^.]+$|$)/i;
|
||||
const stripId = (seg) => seg.replace(NOTION_ID, '');
|
||||
const badChars = /[<>:"|?*\u0000-\u001f]/g;
|
||||
const safeSeg = (s) => s.replace(badChars, '-').replace(/^\.+/, '').trim() || 'Untitled';
|
||||
|
||||
// Where each source path ends up: { 'a/b abc.md': 'a/b.md' }, names made unique.
|
||||
function plan(files) {
|
||||
const map = new Map();
|
||||
const taken = new Set();
|
||||
for (const rel of files.slice().sort()) {
|
||||
const parts = rel.split('/').map((p) => safeSeg(stripId(p)));
|
||||
let target = parts.join('/');
|
||||
const ext = path.posix.extname(target);
|
||||
const stem = target.slice(0, target.length - ext.length);
|
||||
for (let n = 2; taken.has(target.toLowerCase()); n++) target = `${stem} ${n}${ext}`;
|
||||
taken.add(target.toLowerCase());
|
||||
map.set(rel, target);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// Rewrite relative links in one note: to other notes -> [[wiki links]], to files -> the new relative path.
|
||||
function rewriteBody(body, fromRel, map, rootName) {
|
||||
const dir = path.posix.dirname(fromRel);
|
||||
const resolve = (href) => {
|
||||
let h = href.split('#')[0];
|
||||
try { h = decodeURIComponent(h); } catch { /* leave as written */ }
|
||||
if (!h || /^[a-z][a-z0-9+.-]*:/i.test(h) || h.startsWith('/') || h.startsWith('#')) return null;
|
||||
const joined = path.posix.normalize(dir === '.' ? h : `${dir}/${h}`);
|
||||
return joined.startsWith('..') ? null : joined;
|
||||
};
|
||||
return body.replace(/(!?)\[([^\]\n]*)\]\(([^)\n]+)\)/g, (all, bang, text, href) => {
|
||||
const target = resolve(href.trim().replace(/^<|>$/g, ''));
|
||||
if (!target || !map.has(target)) return all;
|
||||
const to = map.get(target);
|
||||
if (!bang && /\.md$/i.test(to)) {
|
||||
const id = `${rootName}/${to.replace(/\.md$/i, '')}`;
|
||||
const title = id.split('/').pop();
|
||||
return text && text !== title ? `[[${id}|${text}]]` : `[[${id}]]`;
|
||||
}
|
||||
// a picture or file: its path from this note's new folder
|
||||
const fromDir = path.posix.dirname(map.get(fromRel));
|
||||
const rel = path.posix.relative(fromDir === '.' ? '' : fromDir, to);
|
||||
return `${bang}[${text}](${encodeURI(rel).replace(/\(/g, '%28').replace(/\)/g, '%29')})`;
|
||||
});
|
||||
}
|
||||
|
||||
// files: [{ rel, read() -> Buffer }]. Writes them under <vaultRoot>/<name>. Returns { folder, notes, files }.
|
||||
async function importFiles(vaultRoot, name, files) {
|
||||
const kept = files.filter((f) => !f.rel.split('/').some((p) => SKIP_DIR.test(p)) && !/_all\.csv$/i.test(f.rel) && !/(^|\/)\.DS_Store$/.test(f.rel));
|
||||
// a zip often has one folder around everything: look inside it
|
||||
const first = new Set(kept.map((f) => f.rel.split('/')[0]));
|
||||
const strip = first.size === 1 && kept.every((f) => f.rel.includes('/')) ? [...first][0] + '/' : '';
|
||||
const items = kept.map((f) => ({ ...f, rel: f.rel.slice(strip.length) })).filter((f) => f.rel);
|
||||
if (!items.length) throw new Error('There was nothing to import in that.');
|
||||
const map = plan(items.map((f) => f.rel));
|
||||
|
||||
let folder = safeSeg(name);
|
||||
for (let n = 2; fs.existsSync(path.join(vaultRoot, folder)) || fs.existsSync(path.join(vaultRoot, folder + '.md')); n++) folder = `${safeSeg(name)} ${n}`;
|
||||
|
||||
let notes = 0, others = 0;
|
||||
for (const f of items) {
|
||||
const to = path.join(vaultRoot, folder, ...map.get(f.rel).split('/'));
|
||||
await fsp.mkdir(path.dirname(to), { recursive: true });
|
||||
let data = f.read();
|
||||
if (/\.md$/i.test(f.rel)) {
|
||||
data = Buffer.from(rewriteBody(data.toString('utf8'), f.rel, map, folder), 'utf8');
|
||||
notes++;
|
||||
} else others++;
|
||||
await fsp.writeFile(to, data);
|
||||
}
|
||||
return { folder, notes, files: others };
|
||||
}
|
||||
|
||||
async function importFolder(vaultRoot, dir) {
|
||||
const files = [];
|
||||
for await (const rel of walkDir(dir)) {
|
||||
if (files.length >= MAX_FILES) break;
|
||||
const full = path.join(dir, rel);
|
||||
files.push({ rel, read: () => fs.readFileSync(full) });
|
||||
}
|
||||
return importFiles(vaultRoot, path.basename(dir), files);
|
||||
}
|
||||
|
||||
async function importZip(vaultRoot, file) {
|
||||
const entries = readZip(file).filter((e) => !e.rel.includes('..'));
|
||||
return importFiles(vaultRoot, path.basename(file).replace(/\.zip$/i, ''), entries);
|
||||
}
|
||||
|
||||
module.exports = { importFolder, importZip, importFiles, readZip, stripId, plan, rewriteBody };
|
||||
Reference in new issue
Block a user