Add a package manager: rex init/add/remove/get/update/deps

Packages come from git (a registry can be added later; a bare version in
rex.toml is reserved for it). rex.toml [deps] names them, rex.lock pins exact
commits, and run/build/check/test download what is missing. `import json`
reads the package's json.rex; package files get `@json...` module keys so they
never mix with the project's own files. Newest tag of the same major version
wins when packages disagree. The language server reads downloaded packages.
Docs: docs/14-packages.md.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Your NameandClaude Opus 5.5 committed 2026-11-16 10:30:11 +01:00
1 parent 300d767124
commit 15da1e2379
16 files changed
+1588 -61

No files matched your search

+289
View File
@@ -0,0 +1,289 @@
//! Packages: `rex add/remove/get/update/deps`, `rex.toml` / `rex.lock`, and imports that reach
//! into packages. Git repositories are made on the fly and used through `file://` addresses;
//! the package cache goes to a temp folder (`REX_HOME`).
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
use rex::loader::Scope;
use rex::pkg::{edit_manifest, newest_tag, parse_manifest, parse_source, Source};
fn temp_dir() -> PathBuf {
static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let dir = std::env::temp_dir().join(format!("rex-packages-{}-{n}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
fn write(dir: &Path, files: &[(&str, &str)]) {
for (name, text) in files {
let f = dir.join(name);
std::fs::create_dir_all(f.parent().unwrap()).unwrap();
std::fs::write(f, text).unwrap();
}
}
fn git(dir: &Path, args: &[&str]) {
let o = Command::new("git")
.args(args)
.current_dir(dir)
.env("GIT_AUTHOR_NAME", "t")
.env("GIT_AUTHOR_EMAIL", "t@t")
.env("GIT_COMMITTER_NAME", "t")
.env("GIT_COMMITTER_EMAIL", "t@t")
.output()
.unwrap();
assert!(o.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&o.stderr));
}
/// Commit everything in `dir` (a new repository the first time) and tag it.
fn release(dir: &Path, files: &[(&str, &str)], tag: Option<&str>) {
std::fs::create_dir_all(dir).unwrap();
if !dir.join(".git").exists() {
git(dir, &["init", "-q"]);
}
write(dir, files);
git(dir, &["add", "."]);
git(dir, &["commit", "-q", "--allow-empty", "-m", "x"]);
if let Some(t) = tag {
git(dir, &["tag", t]);
}
}
fn url(dir: &Path) -> String {
format!("file://{}", dir.display())
}
struct Env {
root: PathBuf,
}
impl Env {
fn new() -> Env {
let root = temp_dir();
std::fs::create_dir_all(root.join("app")).unwrap();
Env { root }
}
fn rex(&self, args: &[&str]) -> Output {
Command::new(env!("CARGO_BIN_EXE_rex"))
.args(args)
.current_dir(self.root.join("app"))
.env("REX_HOME", self.root.join("cache"))
.output()
.unwrap()
}
fn ok(&self, args: &[&str]) -> String {
let o = self.rex(args);
assert!(o.status.success(), "rex {args:?}:\n{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr));
String::from_utf8_lossy(&o.stdout).to_string()
}
fn fails(&self, args: &[&str]) -> String {
let o = self.rex(args);
assert!(!o.status.success(), "rex {args:?} should fail:\n{}", String::from_utf8_lossy(&o.stdout));
String::from_utf8_lossy(&o.stderr).to_string()
}
fn app(&self, files: &[(&str, &str)]) {
write(&self.root.join("app"), files);
}
fn read(&self, file: &str) -> String {
std::fs::read_to_string(self.root.join("app").join(file)).unwrap_or_default()
}
}
impl Drop for Env {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
}
}
/// `strs` (v1.0.0, v1.1.0) with its own `util.rex`, and `json` (v0.3.0), which uses strs v1.0.0.
fn two_packages(e: &Env) -> (PathBuf, PathBuf) {
let strs = e.root.join("strs");
release(
&strs,
&[
("rex.toml", "[package]\nname = \"strs\"\n"),
("strs.rex", "import util\n\npub func shout(s: string): string { return util.bang(s.upper()) }\n"),
("util.rex", "pub func bang(s: string): string { return s + \"!\" }\n"),
],
Some("v1.0.0"),
);
release(&strs, &[("util.rex", "pub func bang(s: string): string { return s + \"!!\" }\n")], Some("v1.1.0"));
let json = e.root.join("json");
release(
&json,
&[
("rex.toml", &format!("[package]\nname = \"json\"\n\n[deps]\nstrs = \"{}@v1.0.0\"\n", url(&strs))),
("json.rex", "import strs\nimport parser\n\npub func hello(): string { return strs.shout(parser.word()) }\n"),
("parser.rex", "pub func word(): string { return \"hi\" }\n"),
],
Some("v0.3.0"),
);
(strs, json)
}
#[test]
fn add_a_package_and_use_it() {
let e = Env::new();
let (strs, json) = two_packages(&e);
e.ok(&["init"]);
let out = e.ok(&["add", &url(&json)]);
assert!(out.contains("added json v0.3.0"), "{out}");
assert!(e.read("rex.toml").contains(&format!("json = \"{}@v0.3.0\"", url(&json))), "{}", e.read("rex.toml"));
let lock = e.read("rex.lock");
assert!(lock.contains(&format!("json {} v0.3.0 ", url(&json))) && lock.contains(&format!("strs {} v1.0.0 ", url(&strs))), "{lock}");
// the project's own util.rex and the package's util.rex are different files
e.app(&[
("main.rex", "import json\nimport util\n\nfunc main() {\n print(json.hello(), util.mine())\n}\n"),
("util.rex", "pub func mine(): int { return 5 }\n"),
]);
assert_eq!(e.ok(&["run", "main.rex"]), "HI! 5\n");
// asking for a newer strs moves json to it too
let out = e.ok(&["add", &format!("{}@v1.1.0", url(&strs))]);
assert!(out.contains("added strs v1.1.0"), "{out}");
assert_eq!(e.ok(&["run", "main.rex"]), "HI!! 5\n");
assert_eq!(e.ok(&["deps"]), "json v0.3.0\n strs v1.1.0\nstrs v1.1.0\n");
e.ok(&["remove", "strs"]);
assert!(!e.read("rex.toml").contains("strs"));
assert_eq!(e.ok(&["run", "main.rex"]), "HI! 5\n");
}
#[test]
fn the_lock_file_keeps_the_build_the_same() {
let e = Env::new();
let lib = e.root.join("lib");
release(&lib, &[("lib.rex", "pub func n(): int { return 1 }\n")], None);
e.ok(&["init"]);
let out = e.ok(&["add", &url(&lib)]);
assert!(out.contains("added lib main branch @"), "{out}");
e.app(&[("main.rex", "import lib\n\nfunc main() { print(lib.n()) }\n")]);
assert_eq!(e.ok(&["run", "main.rex"]), "1\n");
// new commits on the branch don't change the build until `rex update`
release(&lib, &[("lib.rex", "pub func n(): int { return 2 }\n")], None);
assert_eq!(e.ok(&["run", "main.rex"]), "1\n");
let out = e.ok(&["update"]);
assert!(out.contains("lib -> main branch @"), "{out}");
assert_eq!(e.ok(&["run", "main.rex"]), "2\n");
}
#[test]
fn update_moves_to_the_newest_tag_of_the_same_major_version() {
let e = Env::new();
let (strs, _) = two_packages(&e);
e.ok(&["init"]);
e.ok(&["add", &format!("{}@v1.0.0", url(&strs))]);
release(&strs, &[], Some("v2.0.0"));
let out = e.ok(&["update", "strs"]);
assert!(out.contains("strs v1.0.0 -> v1.1.0") && out.contains("v2.0.0 is out too"), "{out}");
assert!(e.read("rex.toml").contains("@v1.1.0"));
assert!(e.ok(&["update"]).contains("v2.0.0 is out too"));
}
#[test]
fn a_package_in_a_folder() {
let e = Env::new();
write(&e.root.join("mylib"), &[("mylib.rex", "import helper\n\npub func n(): int { return helper.k() }\n"), ("helper.rex", "pub func k(): int { return 42 }\n")]);
e.ok(&["init"]);
assert!(e.ok(&["add", "../mylib"]).contains("added mylib (folder ../mylib)"));
assert!(e.read("rex.toml").contains("mylib = \"../mylib\""));
assert!(!e.read("rex.lock").contains("mylib"));
e.app(&[("main.rex", "import mylib\n\nfunc main() { print(mylib.n()) }\n")]);
assert_eq!(e.ok(&["run", "main.rex"]), "42\n");
// changes in the folder show up right away
write(&e.root.join("mylib"), &[("helper.rex", "pub func k(): int { return 7 }\n")]);
assert_eq!(e.ok(&["run", "main.rex"]), "7\n");
}
#[test]
fn package_mistakes_are_explained() {
let e = Env::new();
let (strs, json) = two_packages(&e);
e.ok(&["init"]);
let err = e.fails(&["add", &format!("{}@v9.0.0", url(&strs))]);
assert!(err.contains("no tag, branch or commit called `v9.0.0`") && err.contains("newest version is v1.1.0"), "{err}");
assert!(!e.read("rex.toml").contains("strs"));
e.ok(&["add", &url(&json)]);
// a package that only json uses
e.app(&[("main.rex", "import strs\n\nfunc main() { }\n")]);
let err = e.fails(&["run", "main.rex"]);
assert!(err.contains("`strs` is a package that one of your packages uses"), "{err}");
// a file of your own with the same name as a package
e.app(&[("main.rex", "import json\n\nfunc main() { }\n"), ("json.rex", "pub func x() {}\n")]);
let err = e.fails(&["run", "main.rex"]);
assert!(err.contains("both your file") && err.contains("the package `json`"), "{err}");
std::fs::remove_file(e.root.join("app/json.rex")).unwrap();
// a missing file inside a package
e.app(&[("main.rex", "import json.nope\n\nfunc main() { }\n")]);
let err = e.fails(&["run", "main.rex"]);
assert!(err.contains("the package `json` has no file nope.rex"), "{err}");
// a version without a git address (kept for a registry)
e.app(&[("rex.toml", "[deps]\njson = \"1.2.0\"\n")]);
let err = e.fails(&["run", "main.rex"]);
assert!(err.contains("there is no package registry yet"), "{err}");
}
#[test]
fn two_major_versions_of_one_package_are_an_error() {
let e = Env::new();
let (strs, json) = two_packages(&e);
release(&strs, &[], Some("v2.0.0"));
e.ok(&["init"]);
e.ok(&["add", &url(&json)]);
let err = e.fails(&["add", &format!("{}@v2.0.0", url(&strs))]);
assert!(err.contains("two different packages called `strs`") && err.contains("v1.0.0") && err.contains("v2.0.0"), "{err}");
assert!(!e.read("rex.toml").contains("strs"), "rex.toml is left as it was");
}
#[test]
fn import_keys() {
let mut s = Scope::default();
s.root_deps = vec!["json".into()];
s.packages.insert("json".into(), vec!["strs".into()]);
s.packages.insert("strs".into(), vec![]);
assert_eq!(s.key("", "json").unwrap(), "@json");
assert_eq!(s.key("", "json.parser").unwrap(), "@json.parser");
assert_eq!(s.key("", "util").unwrap(), "util");
assert_eq!(s.key("", "strs").unwrap(), "strs"); // not the project's own package
assert_eq!(s.key("@json", "parser").unwrap(), "@json.parser");
assert_eq!(s.key("@json.parser", "json").unwrap(), "@json");
assert_eq!(s.key("@json", "strs").unwrap(), "@strs");
assert_eq!(s.key("@strs", "util").unwrap(), "@strs.util");
assert!(s.key("@json", "..up").is_err());
}
#[test]
fn reading_and_editing_rex_toml() {
let m = parse_manifest("# mine\n[package]\nname = \"game\" # the name\n\n[deps]\njson = \"a.com/b/json@v1.0.0\"\n", "rex.toml").unwrap();
assert_eq!(m.name.as_deref(), Some("game"));
assert_eq!(m.deps, vec![("json".to_string(), "a.com/b/json@v1.0.0".to_string(), 6)]);
assert!(parse_manifest("[deps]\njson = 1\n", "rex.toml").unwrap_err().contains("rex.toml:2"));
let t = "[package]\nname = \"g\"\n\n[deps]\njson = \"x/y/json\"\n\n[other]\nk = \"v\"\n";
let added = edit_manifest(t, "strs", Some("x/y/strs@v1"));
assert_eq!(added, "[package]\nname = \"g\"\n\n[deps]\njson = \"x/y/json\"\nstrs = \"x/y/strs@v1\"\n\n[other]\nk = \"v\"\n");
assert_eq!(edit_manifest(&added, "json", Some("x/y/json@v2")).lines().nth(4), Some("json = \"x/y/json@v2\""));
assert_eq!(edit_manifest(&added, "strs", None), t);
assert_eq!(edit_manifest("", "a", Some("x/a")), "[deps]\na = \"x/a\"\n");
assert_eq!(edit_manifest("[package]\nname = \"g\"\n", "a", Some("x/a")), "[package]\nname = \"g\"\n\n[deps]\na = \"x/a\"\n");
}
#[test]
fn where_packages_come_from() {
let here = Path::new("/p");
let g = |u: &str, r: Option<&str>| Source::Git { url: u.into(), rev: r.map(Into::into) };
assert_eq!(parse_source("git.joji.li/joji/json@v1.2.0", here).unwrap(), g("git.joji.li/joji/json", Some("v1.2.0")));
assert_eq!(parse_source("github.com/a/b.git", here).unwrap(), g("github.com/a/b", None));
assert_eq!(parse_source("git@host.com:a/b@main", here).unwrap(), g("git@host.com:a/b", Some("main")));
assert_eq!(parse_source("git@host.com:a/b", here).unwrap(), g("git@host.com:a/b", None));
assert_eq!(parse_source("../lib", here).unwrap(), Source::Path(PathBuf::from("/p/../lib")));
assert!(parse_source("1.2.0", here).unwrap_err().contains("registry"));
assert!(parse_source("json", here).is_err());
let tags: Vec<String> = ["v1.0.0", "v1.10.0", "v1.9.0", "v2.0.0-beta.1", "junk"].iter().map(|s| s.to_string()).collect();
assert_eq!(newest_tag(&tags, None).as_deref(), Some("v1.10.0"));
assert_eq!(newest_tag(&tags, Some(2)).as_deref(), Some("v2.0.0-beta.1"));
}