[PATCH radicle-link v1 0/1] add integration test for bins package

This thread contains a patchset. You're looking at the original emails, but you may wish to use the patch review UI. Review patch

59 6

[PATCH radicle-link v1 0/1] add integration test for bins package

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220810054116.2487-1-keepsimple@gmail.com>
DKIM signature: missing

Download raw message

2 years ago

This is the first version patch to add an integration test for bins package. The test 
is following the steps in https://github.com/alexjg/linkd-playground. This version covers the steps
before we git push the local changes and do git clone.

This test is not embedded in CI workflow yet. Run `cargo test` (recommend `cargo test -- --nocapture`)
under bins to execute the test.


Han Xu (1):
  basic integration test for bins

 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 ++
 bins/tests/integration_test.rs | 392 +++++++++++++++++++++++++++++++++
 4 files changed, 435 insertions(+)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v1 1/1] basic integration test for bins

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220810054116.2487-2-keepsimple@gmail.com>
In-Reply-To: <20220810054116.2487-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +435 -0

---
 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 ++
 bins/tests/integration_test.rs | 392 +++++++++++++++++++++++++++++++++
 4 files changed, 435 insertions(+)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

diff --git a/bins/Cargo.lock b/bins/Cargo.lock
index d5756144..4d0ea413 100644
--- a/bins/Cargo.lock
@@ -838,6 +838,17 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "errno"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e2b2decb0484e15560df3210cf0d78654bb0864b2c138977c07e377a1bae0e2"
+dependencies = [
+ "kernel32-sys",
+ "libc",
+ "winapi 0.2.8",
+]
+
 [[package]]
 name = "event-listener"
 version = "2.5.2"
@@ -2945,6 +2956,16 @@ dependencies = [
  "human_format",
 ]
 
+[[package]]
+name = "pty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50f3d255966981eb4e4c5df3e983e6f7d163221f547406d83b6a460ff5c5ee8"
+dependencies = [
+ "errno",
+ "libc",
+]
+
 [[package]]
 name = "quanta"
 version = "0.4.1"
@@ -3567,6 +3588,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "tests"
+version = "0.1.0"
+dependencies = [
+ "pty",
+ "serde_json",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
diff --git a/bins/Cargo.toml b/bins/Cargo.toml
index 3d1786f4..99b3c69f 100644
--- a/bins/Cargo.toml
@@ -5,4 +5,5 @@ members = [
   "lnk-gitd",
   "lnk-identities-dev",
   "lnk-profile-dev",
+  "tests",
 ]
diff --git a/bins/tests/Cargo.toml b/bins/tests/Cargo.toml
new file mode 100644
index 00000000..3ae82769
--- /dev/null
@@ -0,0 +1,13 @@
+[package]
+name = "tests"
+version = "0.1.0"
+authors = ["Han Xu <keepsimple@gmail.com>"]
+edition = "2021"
+
+[dev-dependencies]
+pty = "0.2"
+serde_json = "1.0"
+
+[[test]]
+name = "integration_test"
+path = "integration_test.rs"
diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
new file mode 100644
index 00000000..4ff581a0
--- /dev/null
@@ -0,0 +1,392 @@
+// Copyright © 2022 The Radicle Link Contributors
+//
+// This file is part of radicle-link, distributed under the GPLv3 with Radicle
+// Linking Exception. For full terms see the included LICENSE file.
+
+//! Integration test to exercise the programs in `bins`.
+
+use pty::fork::*;
+use serde_json::{json, Value};
+use std::{
+    env,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    process::{Child, Command, Stdio},
+    thread,
+    time::{Duration, SystemTime},
+};
+
+/// This test is inspired by https://github.com/alexjg/linkd-playground
+#[test]
+fn happy_path_to_push_changes() {
+    let peer1_home = "/tmp/link-local-1";
+    let peer2_home = "/tmp/link-local-2";
+    let seed_home = "/tmp/seed-home";
+    let passphrase = b"play\n";
+
+    println!("\n== create lnk homes for two peers and one seed ==\n");
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Creating local link 1 identity ==\n");
+    let peer1_name = "sockpuppet1".to_string();
+    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer1_name), peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let urn1 = v["urn"].as_str().unwrap().to_string();
+    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn1), peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Creating local link 2 identity ==\n");
+    let peer2_name = "sockpuppet2".to_string();
+    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer2_name), peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let urn2 = v["urn"].as_str().unwrap().to_string();
+    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn2), peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Create a local repository ==\n");
+    let peer1_proj = format!("peer1_proj_{}", timestamp());
+    let (is_parent, output) = run_lnk(
+        LnkCmd::IdProjectCreate(peer1_proj.clone()),
+        peer1_home,
+        passphrase,
+    );
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let proj_urn = v["urn"].as_str().unwrap().to_string();
+    println!("our project URN: {}", &proj_urn);
+
+    println!("\n== Add the seed to the local peer seed configs ==\n");
+    let (is_parent, seed_peer_id) = run_lnk(LnkCmd::ProfilePeer, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let seed_endpoint = format!("{}@127.0.0.1:8799", &seed_peer_id);
+
+    let (is_parent, peer1_profile) = run_lnk(LnkCmd::ProfileGet, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
+    let mut peer1_f = File::create(peer1_seed).unwrap();
+    peer1_f.write_all(seed_endpoint.as_bytes()).unwrap();
+
+    let (is_parent, peer2_profile) = run_lnk(LnkCmd::ProfileGet, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
+    let mut peer2_f = File::create(peer2_seed).unwrap();
+    peer2_f.write_all(seed_endpoint.as_bytes()).unwrap();
+
+    println!("\n== Start the seed linkd ==\n");
+    let manifest_path = manifest_path();
+    let mut linkd = spawn_linkd(seed_home, &manifest_path);
+
+    println!("\n== Start the peer 1 gitd ==\n");
+    let (is_parent, peer1_peer_id) = run_lnk(LnkCmd::ProfilePeer, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let mut lnk_gitd = spawn_lnk_gitd(peer1_home, &manifest_path, &peer1_peer_id);
+
+    println!("\n== Make some changes in the repo ==\n");
+    env::set_current_dir(&peer1_proj).unwrap();
+    let mut test_file = File::create("test").unwrap();
+    test_file.write_all(b"test").unwrap();
+    Command::new("git")
+        .arg("add")
+        .arg("test")
+        .output()
+        .expect("failed to do git add");
+    let output = Command::new("git")
+        .arg("commit")
+        .arg("-m")
+        .arg("test commit")
+        .output()
+        .expect("failed to do git commit");
+    println!("git-commit: {:?}", &output);
+
+    println!("\n== Add the linkd remote to the repo ==\n");
+    let remote_url = format!("ssh://rad@127.0.0.1:9987/{}.git", &proj_urn);
+    Command::new("git")
+        .arg("remote")
+        .arg("add")
+        .arg("linkd")
+        .arg(remote_url)
+        .output()
+        .expect("failed to do git remote add");
+
+    clean_up_known_hosts();
+
+    linkd.kill().ok();
+    lnk_gitd.kill().ok();
+}
+
+enum LnkCmd {
+    ProfileCreate,
+    ProfileGet,
+    ProfilePeer,
+    ProfileSshAdd,
+    IdPersonCreate(String),  // the associated string is "the person's name".
+    IdLocalSet(String),      // the associated string is "urn".
+    IdProjectCreate(String), // the associated string is "the project name".
+}
+
+/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
+/// Return.0: true if this is the parent (i.e. test) process,
+///           false if this is the child (i.e. lnk) process.
+/// Return.1: an output that depends on the `cmd`.
+fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        // Input the passphrase if necessary.
+        match cmd {
+            LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
+                parent.write_all(passphrase).unwrap();
+                println!("{}: wrote passphase", lnk_home);
+            },
+            _ => {},
+        }
+
+        // Print the output and decode them if necessary.
+        let buf_reader = BufReader::new(parent);
+        let mut output = String::new();
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("{}: {}", lnk_home, line);
+
+            match cmd {
+                LnkCmd::IdPersonCreate(ref _name) => {
+                    if line.find("\"urn\":").is_some() {
+                        output = line; // get the line with URN.
+                    }
+                },
+                LnkCmd::IdProjectCreate(ref _name) => {
+                    if line.find("\"urn\":").is_some() {
+                        output = line; // get the line with URN.
+                    }
+                },
+                LnkCmd::ProfileGet => {
+                    output = line; // get the last line for profile id.
+                },
+                LnkCmd::ProfilePeer => {
+                    output = line; // get the last line for peer id.
+                },
+                _ => {},
+            }
+        }
+
+        (true, output)
+    } else {
+        // Child process is to run `lnk`.
+        let manifest_path = manifest_path();
+
+        // cargo run \
+        // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
+        // -p lnk -- "$@"
+        let mut lnk_cmd = Command::new("cargo");
+        lnk_cmd
+            .env("LNK_HOME", lnk_home)
+            .arg("run")
+            .arg("--manifest-path")
+            .arg(manifest_path)
+            .arg("-p")
+            .arg("lnk")
+            .arg("--");
+        let full_cmd = match cmd {
+            LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
+            LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
+            LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
+            LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
+            LnkCmd::IdPersonCreate(name) => {
+                let payload = json!({ "name": name });
+                lnk_cmd
+                    .arg("identities")
+                    .arg("person")
+                    .arg("create")
+                    .arg("new")
+                    .arg("--payload")
+                    .arg(payload.to_string())
+            },
+            LnkCmd::IdLocalSet(urn) => lnk_cmd
+                .arg("identities")
+                .arg("local")
+                .arg("set")
+                .arg("--urn")
+                .arg(urn),
+            LnkCmd::IdProjectCreate(name) => {
+                let payload = json!({"name": name, "default_branch": "master"});
+                let project_path = format!("./{}", name);
+                lnk_cmd
+                    .arg("identities")
+                    .arg("project")
+                    .arg("create")
+                    .arg("new")
+                    .arg("--path")
+                    .arg(project_path)
+                    .arg("--payload")
+                    .arg(payload.to_string())
+            },
+        };
+        full_cmd.status().expect("lnk cmd failed:");
+
+        (false, String::new())
+    }
+}
+
+fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
+    let log_name = format!("linkd_{}.log", &timestamp());
+    let log_file = File::create(&log_name).unwrap();
+    let child = Command::new("cargo")
+        .arg("run")
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("linkd")
+        .arg("--")
+        .arg("--lnk-home")
+        .arg(lnk_home)
+        .arg("--track")
+        .arg("everything")
+        .arg("--protocol-listen")
+        .arg("127.0.0.1:8799")
+        .stdout(Stdio::from(log_file))
+        .spawn()
+        .expect("linkd failed to start");
+    println!("linkd stdout redirected to {}", &log_name);
+    thread::sleep(Duration::from_secs(1));
+    child
+}
+
+fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) -> Child {
+    let log_name = format!("lnk-gitd_{}.log", &timestamp());
+    let log_file = File::create(&log_name).unwrap();
+    let port = "9987";
+    let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
+    let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg("./target")
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk-gitd")
+        .output()
+        .expect("cargo build lnk-gitd failed");
+
+    let child = Command::new("systemd-socket-activate")
+        .arg("-l")
+        .arg(port)
+        .arg("--fdname=ssh")
+        .arg("-E")
+        .arg("SSH_AUTH_SOCK")
+        .arg("-E")
+        .arg("RUST_BACKTRACE")
+        .arg("./target/debug/lnk-gitd")
+        .arg(lnk_home)
+        .arg("--linkd-rpc-socket")
+        .arg(rpc_socket)
+        .arg("--push-seeds")
+        .arg("--fetch-seeds")
+        .arg("--linger-timeout")
+        .arg("10000")
+        .stdout(Stdio::from(log_file))
+        .spawn()
+        .expect("lnk-gitd failed to start");
+    println!("lnk-gitd stdout redirected to {}", &log_name);
+    thread::sleep(Duration::from_secs(1));
+
+    child
+}
+
+/// Returns true if this is the parent process,
+/// returns false if this is the child process.
+fn _run_git_push() -> bool {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        let yes = b"yes\n";
+        parent.write_all(yes).unwrap();
+
+        let buf_reader = BufReader::new(parent);
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("git-push: {}", line);
+        }
+
+        true
+    } else {
+        Command::new("git")
+            .arg("push")
+            .arg("linkd")
+            .status()
+            .expect("failed to do git push");
+        false
+    }
+}
+
+/// Returns UNIX_TIME in millis.
+fn timestamp() -> u128 {
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap();
+    now.as_millis()
+}
+
+/// Returns the full path of `bins` manifest file.
+fn manifest_path() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/Cargo.toml", package_dir.strip_suffix("/tests").unwrap())
+}
+
+fn clean_up_known_hosts() {
+    // ssh-keygen -f "/home/pi/.ssh/known_hosts" -R "[127.0.0.1]:9987"
+    let home_dir = env!("HOME");
+    let known_hosts = format!("{}/.ssh/known_hosts", &home_dir);
+    let output = Command::new("ssh-keygen")
+        .arg("-f")
+        .arg(known_hosts)
+        .arg("-R")
+        .arg("[127.0.0.1]:9987")
+        .output()
+        .expect("failed to do ssh-keygen");
+    println!("ssh-keygen: {:?}", &output);
+}
-- 
2.32.0 (Apple Git-132)

[radicle-link/patches/nixos-latest.yml] build failed

builds.sr.ht <builds@sr.ht>

Details

Message ID: <CM24F8WIYXJ6.1JBFKDD9FQC7Q@cirno2>
In-Reply-To: <20220810054116.2487-2-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

radicle-link/patches/nixos-latest.yml: FAILED in 23m57s

[add integration test for bins package][0] from [Han Xu][1]

[0]: https://lists.sr.ht/~radicle-link/dev/patches/34576
[1]: keepsimple@gmail.com

✗ #820143 FAILED radicle-link/patches/nixos-latest.yml https://builds.sr.ht/~radicle-link/job/820143

[PATCH radicle-link v2 0/1] add integration test for bins package

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220817072940.74222-1-keepsimple@gmail.com>
In-Reply-To: <20220810054116.2487-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Finally I got the full test working, with the same steps as in Alex's 
playground script, except it is done in Rust now and we can just run
`cargo test`. 

Also, thanks to Alex's pointer to a git version check, I found a bug that 
it turns out that git version 2.25.1 (Ubuntu 20.04) or version 2.30.1
(Debian 11) can support our request-pull with no problem. But the Git 
version checking code in link-git/src/protocol/ls.rs and fetch.rs
will insert namespaces into the refs. As the result, `linkd` on the seed
will panic and `request-pull` will fail. 

Please see the test case doc comments for more details. Here is a log of
running this test, so that you can see what the test looks like.

=================  TEST LOG =====================================

radicle-link/bins/tests (integration-test) $ cargo test -- --nocapture
   Compiling tests v0.1.0 (/home/pi/work/radicle-link/bins/tests)
    Finished test [unoptimized + debuginfo] target(s) in 0.69s
     Running integration_test.rs
(/home/pi/work/radicle-link/bins/target/debug/deps/integration_test-52234403479d7335)

running 1 test

== create lnk homes for two peers and one seed ==

/tmp/link-local-1: wrote passphase
/tmp/link-local-1: play
/tmp/link-local-1:    Compiling link-git v0.1.0
(/home/pi/work/radicle-link/link-git)
/tmp/link-local-1:    Compiling radicle-git-ext v0.1.0
(/home/pi/work/radicle-link/git-ext)
/tmp/link-local-1:    Compiling link-crypto v0.1.0
(/home/pi/work/radicle-link/link-crypto)
/tmp/link-local-1:    Compiling radicle-macros v0.1.0
(/home/pi/work/radicle-link/macros)
/tmp/link-local-1:    Compiling link-replication v0.1.0
(/home/pi/work/radicle-link/link-replication)
/tmp/link-local-1:    Compiling link-identities v0.1.0
(/home/pi/work/radicle-link/link-identities)
/tmp/link-local-1:    Compiling cob v0.1.0 (/home/pi/work/radicle-link/cob)
/tmp/link-local-1:    Compiling link-hooks v0.1.0
(/home/pi/work/radicle-link/link-hooks)
/tmp/link-local-1:    Compiling link-tracking v0.1.0
(/home/pi/work/radicle-link/link-tracking)
/tmp/link-local-1:    Compiling librad v0.1.0
(/home/pi/work/radicle-link/librad)
/tmp/link-local-1:    Compiling lnk-clib v0.1.0
(/home/pi/work/radicle-link/cli/lnk-clib)
/tmp/link-local-1:    Compiling lnk-identities v0.1.0
(/home/pi/work/radicle-link/cli/lnk-identities)
/tmp/link-local-1:    Compiling lnk-profile v0.1.0
(/home/pi/work/radicle-link/cli/lnk-profile)
/tmp/link-local-1:    Compiling lnk-sync v0.1.0
(/home/pi/work/radicle-link/cli/lnk-sync)
/tmp/link-local-1:    Compiling lnk-exe v0.1.0
(/home/pi/work/radicle-link/cli/lnk-exe)
/tmp/link-local-1:    Compiling lnk v0.1.0 (/home/pi/work/radicle-link/bins/lnk)
/tmp/link-local-1:     Finished dev [unoptimized + debuginfo]
target(s) in 44.43s
/tmp/link-local-1:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk profile create`
/tmp/link-local-1: please enter your passphrase: profile id:
0cd616e9-1c18-4b32-857c-470f1e719a98
/tmp/link-local-1: peer id:
hydo86fxtnqwcy3o7ut5gypre4nc1a551u9jdkoonzpjxkphfwwqiq
/tmp/link-local-2: wrote passphase
/tmp/link-local-2: play
/tmp/link-local-2:     Finished dev [unoptimized + debuginfo] target(s) in 0.26s
/tmp/link-local-2:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk profile create`
/tmp/link-local-2: please enter your passphrase: profile id:
9d69fbe0-7d32-40d5-bcb5-2f3164682648
/tmp/link-local-2: peer id:
hyn4ycegfgq369pczgcc5i36ztmopmwy4rktnok4cg69jowdd1oejg
/tmp/seed-home: wrote passphase
/tmp/seed-home: play
/tmp/seed-home:     Finished dev [unoptimized + debuginfo] target(s) in 0.17s
/tmp/seed-home:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk profile create`
test two_peers_and_a_seed has been running for over 60 seconds
/tmp/seed-home: please enter your passphrase: profile id:
e049dbbb-b426-4fd2-8f96-d863f2aa7ff9
/tmp/seed-home: peer id: hybodew66mqgue1t7a976bznwtbs7fhu3f8m9461m3fdhfbdwhqa8w

== add ssh keys for each profile to the ssh-agent ==

/tmp/link-local-1: wrote passphase
/tmp/link-local-1: play
/tmp/link-local-1:     Finished dev [unoptimized + debuginfo] target(s) in 0.16s
/tmp/link-local-1:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk profile ssh add`
/tmp/link-local-1: please enter your passphrase: added key for profile
id `0cd616e9-1c18-4b32-857c-470f1e719a98`
/tmp/link-local-2: wrote passphase
/tmp/link-local-2: play
/tmp/link-local-2:     Finished dev [unoptimized + debuginfo] target(s) in 0.18s
/tmp/link-local-2:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk profile ssh add`
/tmp/link-local-2: please enter your passphrase: added key for profile
id `9d69fbe0-7d32-40d5-bcb5-2f3164682648`
/tmp/seed-home: wrote passphase
/tmp/seed-home: play
/tmp/seed-home:     Finished dev [unoptimized + debuginfo] target(s) in 0.17s
/tmp/seed-home:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk profile ssh add`
/tmp/seed-home: please enter your passphrase: added key for profile id
`e049dbbb-b426-4fd2-8f96-d863f2aa7ff9`

== Creating local link 1 identity ==

/tmp/link-local-1:     Finished dev [unoptimized + debuginfo] target(s) in 0.17s
/tmp/link-local-1:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk identities person
create new --payload '{"name":"sockpuppet1"}'`
/tmp/link-local-1:
{"urn":"rad:git:hnrkqmwrob5fb8sae9egn1stciqhj5tjcicwo","payload":{"https://radicle.xyz/link/identities/person/v1":{"name":"sockpuppet1"}}}
/tmp/link-local-1:     Finished dev [unoptimized + debuginfo] target(s) in 0.16s
/tmp/link-local-1:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk identities local set
--urn 'rad:git:hnrkqmwrob5fb8sae9egn1stciqhj5tjcicwo'`
/tmp/link-local-1: set default identity to
`rad:git:hnrkqmwrob5fb8sae9egn1stciqhj5tjcicwo`

== Creating local link 2 identity ==

/tmp/link-local-2:     Finished dev [unoptimized + debuginfo] target(s) in 0.16s
/tmp/link-local-2:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk identities person
create new --payload '{"name":"sockpuppet2"}'`
/tmp/link-local-2:
{"urn":"rad:git:hnrkntsedajm1hoy7zizz8nytqqf8jt3ng3po","payload":{"https://radicle.xyz/link/identities/person/v1":{"name":"sockpuppet2"}}}
/tmp/link-local-2:     Finished dev [unoptimized + debuginfo] target(s) in 0.18s
/tmp/link-local-2:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk identities local set
--urn 'rad:git:hnrkntsedajm1hoy7zizz8nytqqf8jt3ng3po'`
/tmp/link-local-2: set default identity to
`rad:git:hnrkntsedajm1hoy7zizz8nytqqf8jt3ng3po`

== Create a local repository ==

/tmp/link-local-1:     Finished dev [unoptimized + debuginfo] target(s) in 0.17s
/tmp/link-local-1:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk identities project
create new --path ./peer1_proj_1660720324161 --payload
'{"default_branch":"master","name":"peer1_proj_1660720324161"}'`
/tmp/link-local-1:
{"urn":"rad:git:hnrkf7x96s1pwdxux1ee45o1pq6fyk58tixto","payload":{"https://radicle.xyz/link/identities/project/v1":{"name":"peer1_proj_1660720324161","description":null,"default_branch":"master"}}}
our project URN: rad:git:hnrkf7x96s1pwdxux1ee45o1pq6fyk58tixto

== Add the seed to the local peer seed configs ==

/tmp/seed-home:     Finished dev [unoptimized + debuginfo] target(s) in 0.17s
/tmp/seed-home:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk profile peer`
/tmp/seed-home: hybodew66mqgue1t7a976bznwtbs7fhu3f8m9461m3fdhfbdwhqa8w
/tmp/link-local-1:     Finished dev [unoptimized + debuginfo] target(s) in 0.17s
/tmp/link-local-1:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk profile get`
/tmp/link-local-1: 0cd616e9-1c18-4b32-857c-470f1e719a98
/tmp/link-local-2:     Finished dev [unoptimized + debuginfo] target(s) in 0.17s
/tmp/link-local-2:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk profile get`
/tmp/link-local-2: 9d69fbe0-7d32-40d5-bcb5-2f3164682648

== Start the seed linkd ==

linkd stdout redirected to linkd_1660720325160.log

== Start the peer 1 gitd ==

/tmp/link-local-1:     Finished dev [unoptimized + debuginfo] target(s) in 0.66s
/tmp/link-local-1:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk profile peer`
/tmp/link-local-1: hydo86fxtnqwcy3o7ut5gypre4nc1a551u9jdkoonzpjxkphfwwqiq
started lnk-gitd

== Make some changes in the repo ==

git-commit: Output { status: ExitStatus(unix_wait_status(0)), stdout:
"[master 6b7017d] test commit\n 1 file changed, 1 insertion(+)\n
create mode 100644 test\n", stderr: "" }

== Add the linkd remote to the repo ==

ssh-keygen: Output { status: ExitStatus(unix_wait_status(0)), stdout:
"# Host [127.0.0.1]:9987 found: line 1\n/home/pi/.ssh/known_hosts
updated.\nOriginal contents retained as
/home/pi/.ssh/known_hosts.old\n", stderr: "" }
The authenticity of host '[127.0.0.1]:9987 ([127.0.0.1]:9987)' can't
be established.
git-push: ED25519 key fingerprint is
SHA256:s5MnhkyLhIdIdL/2DVDfPheb9JPrfQF0jSgY4qehqMM.
git-push: Are you sure you want to continue connecting
(yes/no/[fingerprint])? yes
git-push: Warning: Permanently added '[127.0.0.1]:9987' (ED25519) to
the list of known hosts.
git-push: Enumerating objects: 4, done.
Counting objects: 100% (4/4), done./4)
Writing objects: 100% (3/3), 239 bytes | 239.00 KiB/s, done.
git-push: Total 3 (delta 0), reused 0 (delta 0)
git-push: request-pull to
`hybodew66mqgue1t7a976bznwtbs7fhu3f8m9461m3fdhfbdwhqa8w`
git-push: Checking if request-pull is allowed for
`rad:git:hnrkf7x96s1pwdxux1ee45o1pq6fyk58tixto`
git-push: Tracked
`refs/rad/remotes/hnrkf7x96s1pwdxux1ee45o1pq6fyk58tixto/hydo86fxtnqwcy3o7ut5gypre4nc1a551u9jdkoonzpjxkphfwwqiq`
git-push: Starting replication for
`rad:git:hnrkf7x96s1pwdxux1ee45o1pq6fyk58tixto`
git-push: updated references:
git-push: +refs/rad/id->b86734e7dcd8c57d1b533b3cdc228626a557a89a
git-push: +refs/namespaces/hnrkqmwrob5fb8sae9egn1stciqhj5tjcicwo/refs/rad/id->77c13533f7359d0903f0111c3c21521f864a0557
git-push: +refs/remotes/hydo86fxtnqwcy3o7ut5gypre4nc1a551u9jdkoonzpjxkphfwwqiq/rad/ids/hnrkqmwrob5fb8sae9egn1stciqhj5tjcicwo->77c13533f7359d0903f0111c3c21521f864a0557
git-push: +refs/rad/ids/hnrkqmwrob5fb8sae9egn1stciqhj5tjcicwo->77c13533f7359d0903f0111c3c21521f864a0557
git-push: +refs/remotes/hydo86fxtnqwcy3o7ut5gypre4nc1a551u9jdkoonzpjxkphfwwqiq/rad/id->b86734e7dcd8c57d1b533b3cdc228626a557a89a
git-push: +refs/remotes/hydo86fxtnqwcy3o7ut5gypre4nc1a551u9jdkoonzpjxkphfwwqiq/heads/master->85d6729f10f248391a8b2b11e02d4a925d874799
git-push: +refs/remotes/hydo86fxtnqwcy3o7ut5gypre4nc1a551u9jdkoonzpjxkphfwwqiq/rad/self->77c13533f7359d0903f0111c3c21521f864a0557
git-push: +refs/remotes/hydo86fxtnqwcy3o7ut5gypre4nc1a551u9jdkoonzpjxkphfwwqiq/rad/signed_refs->a065e7c06c12c28d8ca959eb7eef1d1d5dd5c088
git-push:
git-push: updating signed refs
git-push: signed refs state was updated
git-push: skipping announce, use `--announce-on-push` if you wish to
execute this step
git-push: To ssh://127.0.0.1:9987/rad:git:hnrkf7x96s1pwdxux1ee45o1pq6fyk58tixto.git
git-push:    85d6729..6b7017d  master -> master

== Clone to peer2 ==

/tmp/link-local-2:     Finished dev [unoptimized + debuginfo] target(s) in 0.16s
/tmp/link-local-2:      Running
`/home/pi/work/radicle-link/bins/target/debug/lnk clone --urn
'rad:git:hnrkf7x96s1pwdxux1ee45o1pq6fyk58tixto' --path
peer2_proj_1660720452541 --peer
hydo86fxtnqwcy3o7ut5gypre4nc1a551u9jdkoonzpjxkphfwwqiq`
/tmp/link-local-2: cloning urn
rad:git:hnrkf7x96s1pwdxux1ee45o1pq6fyk58tixto into
peer2_proj_1660720452541
/tmp/link-local-2: syncing monorepo with seeds
/tmp/link-local-2: working copy created at
`/home/pi/work/radicle-link/bins/tests/peer2_proj_1660720452541/.git/`

== Kill linkd (seed) ==

test two_peers_and_a_seed ... ok

test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered
out; finished in 213.72s




Han Xu (1):
  Add integration test for bins

 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 458 +++++++++++++++++++++++++++++++++
 link-git/src/protocol/fetch.rs |   2 +-
 link-git/src/protocol/ls.rs    |   2 +-
 6 files changed, 503 insertions(+), 2 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v2 1/1] Add integration test for bins

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220817072940.74222-2-keepsimple@gmail.com>
In-Reply-To: <20220817072940.74222-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +503 -2

---
 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 458 +++++++++++++++++++++++++++++++++
 link-git/src/protocol/fetch.rs |   2 +-
 link-git/src/protocol/ls.rs    |   2 +-
 6 files changed, 503 insertions(+), 2 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

diff --git a/bins/Cargo.lock b/bins/Cargo.lock
index d5756144..4d0ea413 100644
--- a/bins/Cargo.lock
@@ -838,6 +838,17 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "errno"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e2b2decb0484e15560df3210cf0d78654bb0864b2c138977c07e377a1bae0e2"
+dependencies = [
+ "kernel32-sys",
+ "libc",
+ "winapi 0.2.8",
+]
+
 [[package]]
 name = "event-listener"
 version = "2.5.2"
@@ -2945,6 +2956,16 @@ dependencies = [
  "human_format",
 ]
 
+[[package]]
+name = "pty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50f3d255966981eb4e4c5df3e983e6f7d163221f547406d83b6a460ff5c5ee8"
+dependencies = [
+ "errno",
+ "libc",
+]
+
 [[package]]
 name = "quanta"
 version = "0.4.1"
@@ -3567,6 +3588,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "tests"
+version = "0.1.0"
+dependencies = [
+ "pty",
+ "serde_json",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
diff --git a/bins/Cargo.toml b/bins/Cargo.toml
index 3d1786f4..99b3c69f 100644
--- a/bins/Cargo.toml
@@ -5,4 +5,5 @@ members = [
   "lnk-gitd",
   "lnk-identities-dev",
   "lnk-profile-dev",
+  "tests",
 ]
diff --git a/bins/tests/Cargo.toml b/bins/tests/Cargo.toml
new file mode 100644
index 00000000..3ae82769
--- /dev/null
@@ -0,0 +1,13 @@
+[package]
+name = "tests"
+version = "0.1.0"
+authors = ["Han Xu <keepsimple@gmail.com>"]
+edition = "2021"
+
+[dev-dependencies]
+pty = "0.2"
+serde_json = "1.0"
+
+[[test]]
+name = "integration_test"
+path = "integration_test.rs"
diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
new file mode 100644
index 00000000..d6d3b4ae
--- /dev/null
@@ -0,0 +1,458 @@
+// Copyright © 2022 The Radicle Link Contributors
+//
+// This file is part of radicle-link, distributed under the GPLv3 with Radicle
+// Linking Exception. For full terms see the included LICENSE file.
+
+//! Integration test to exercise the programs in `bins`.
+
+use pty::fork::*;
+use serde_json::{json, Value};
+use std::{
+    env,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    process::{Child, Command, Stdio},
+    thread,
+    time::{Duration, SystemTime},
+};
+
+/// This test is inspired by https://github.com/alexjg/linkd-playground
+///
+/// Tests a typical scenario: there are two peer nodes and one seed node.
+/// The main steps are:
+///   - Setup a profile for each node in tmp home directories.
+///   - Setup SSH keys for each profile.
+///   - Create identities.
+///   - Create a local repo for peer 1.
+///   - Start `linkd` for the seed, and `lnk-gitd` for peer 1.
+///   - Push peer 1 repo to its monorepo and to the seed.
+///   - Clone the peer 1 repo to peer 2 via seed.
+#[test]
+fn two_peers_and_a_seed() {
+    let peer1_home = "/tmp/link-local-1";
+    let peer2_home = "/tmp/link-local-2";
+    let seed_home = "/tmp/seed-home";
+    let passphrase = b"play\n";
+
+    println!("\n== create lnk homes for two peers and one seed ==\n");
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Creating local link 1 identity ==\n");
+    let peer1_name = "sockpuppet1".to_string();
+    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer1_name), peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let urn1 = v["urn"].as_str().unwrap().to_string();
+    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn1), peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Creating local link 2 identity ==\n");
+    let peer2_name = "sockpuppet2".to_string();
+    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer2_name), peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let urn2 = v["urn"].as_str().unwrap().to_string();
+    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn2), peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Create a local repository ==\n");
+    let peer1_proj = format!("peer1_proj_{}", timestamp());
+    let (is_parent, output) = run_lnk(
+        LnkCmd::IdProjectCreate(peer1_proj.clone()),
+        peer1_home,
+        passphrase,
+    );
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let proj_urn = v["urn"].as_str().unwrap().to_string();
+    println!("our project URN: {}", &proj_urn);
+
+    println!("\n== Add the seed to the local peer seed configs ==\n");
+    let (is_parent, seed_peer_id) = run_lnk(LnkCmd::ProfilePeer, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let seed_endpoint = format!("{}@127.0.0.1:8799", &seed_peer_id);
+
+    let (is_parent, peer1_profile) = run_lnk(LnkCmd::ProfileGet, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
+    let mut peer1_f = File::create(peer1_seed).unwrap();
+    peer1_f.write_all(seed_endpoint.as_bytes()).unwrap();
+
+    let (is_parent, peer2_profile) = run_lnk(LnkCmd::ProfileGet, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
+    let mut peer2_f = File::create(peer2_seed).unwrap();
+    peer2_f.write_all(seed_endpoint.as_bytes()).unwrap();
+
+    println!("\n== Start the seed linkd ==\n");
+    let manifest_path = manifest_path();
+    let mut linkd = spawn_linkd(seed_home, &manifest_path);
+
+    println!("\n== Start the peer 1 gitd ==\n");
+    let (is_parent, peer1_peer_id) = run_lnk(LnkCmd::ProfilePeer, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let is_parent = spawn_lnk_gitd(peer1_home, &manifest_path, &peer1_peer_id);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Make some changes in the repo ==\n");
+    env::set_current_dir(&peer1_proj).unwrap();
+    let mut test_file = File::create("test").unwrap();
+    test_file.write_all(b"test").unwrap();
+    Command::new("git")
+        .arg("add")
+        .arg("test")
+        .output()
+        .expect("failed to do git add");
+    let output = Command::new("git")
+        .arg("commit")
+        .arg("-m")
+        .arg("test commit")
+        .output()
+        .expect("failed to do git commit");
+    println!("git-commit: {:?}", &output);
+
+    println!("\n== Add the linkd remote to the repo ==\n");
+    let remote_url = format!("ssh://rad@127.0.0.1:9987/{}.git", &proj_urn);
+    Command::new("git")
+        .arg("remote")
+        .arg("add")
+        .arg("linkd")
+        .arg(remote_url)
+        .output()
+        .expect("failed to do git remote add");
+
+    clean_up_known_hosts();
+
+    let is_parent = run_git_push();
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Clone to peer2 ==\n");
+
+    env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
+    let (is_parent, _) = run_lnk(
+        LnkCmd::Clone(proj_urn, peer1_peer_id),
+        peer2_home,
+        passphrase,
+    );
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Kill linkd (seed) ==\n");
+
+    linkd.kill().ok();
+}
+
+enum LnkCmd {
+    ProfileCreate,
+    ProfileGet,
+    ProfilePeer,
+    ProfileSshAdd,
+    IdPersonCreate(String),  // the associated string is "the person's name".
+    IdLocalSet(String),      // the associated string is "urn".
+    IdProjectCreate(String), // the associated string is "the project name".
+    Clone(String, String),   // the associated string is "urn", "peer_id"
+}
+
+/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
+/// Return.0: true if this is the parent (i.e. test) process,
+///           false if this is the child (i.e. lnk) process.
+/// Return.1: an output that depends on the `cmd`.
+fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        // Input the passphrase if necessary.
+        match cmd {
+            LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
+                parent.write_all(passphrase).unwrap();
+                println!("{}: wrote passphase", lnk_home);
+            },
+            _ => {},
+        }
+
+        // Print the output and decode them if necessary.
+        let buf_reader = BufReader::new(parent);
+        let mut output = String::new();
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("{}: {}", lnk_home, line);
+
+            match cmd {
+                LnkCmd::IdPersonCreate(ref _name) => {
+                    if line.find("\"urn\":").is_some() {
+                        output = line; // get the line with URN.
+                    }
+                },
+                LnkCmd::IdProjectCreate(ref _name) => {
+                    if line.find("\"urn\":").is_some() {
+                        output = line; // get the line with URN.
+                    }
+                },
+                LnkCmd::ProfileGet => {
+                    output = line; // get the last line for profile id.
+                },
+                LnkCmd::ProfilePeer => {
+                    output = line; // get the last line for peer id.
+                },
+                LnkCmd::Clone(ref _urn, ref _peer) => {
+                    output = line;
+                },
+                _ => {},
+            }
+        }
+
+        (true, output)
+    } else {
+        // Child process is to run `lnk`.
+        let manifest_path = manifest_path();
+
+        // cargo run \
+        // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
+        // -p lnk -- "$@"
+        let mut lnk_cmd = Command::new("cargo");
+        lnk_cmd
+            .env("LNK_HOME", lnk_home)
+            .arg("run")
+            .arg("--manifest-path")
+            .arg(manifest_path)
+            .arg("-p")
+            .arg("lnk")
+            .arg("--");
+        let full_cmd = match cmd {
+            LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
+            LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
+            LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
+            LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
+            LnkCmd::IdPersonCreate(name) => {
+                let payload = json!({ "name": name });
+                lnk_cmd
+                    .arg("identities")
+                    .arg("person")
+                    .arg("create")
+                    .arg("new")
+                    .arg("--payload")
+                    .arg(payload.to_string())
+            },
+            LnkCmd::IdLocalSet(urn) => lnk_cmd
+                .arg("identities")
+                .arg("local")
+                .arg("set")
+                .arg("--urn")
+                .arg(urn),
+            LnkCmd::IdProjectCreate(name) => {
+                let payload = json!({"name": name, "default_branch": "master"});
+                let project_path = format!("./{}", name);
+                lnk_cmd
+                    .arg("identities")
+                    .arg("project")
+                    .arg("create")
+                    .arg("new")
+                    .arg("--path")
+                    .arg(project_path)
+                    .arg("--payload")
+                    .arg(payload.to_string())
+            },
+            LnkCmd::Clone(urn, peer_id) => {
+                let peer2_proj = format!("peer2_proj_{}", timestamp());
+                lnk_cmd
+                    .arg("clone")
+                    .arg("--urn")
+                    .arg(urn)
+                    .arg("--path")
+                    .arg(peer2_proj)
+                    .arg("--peer")
+                    .arg(peer_id)
+            },
+        };
+        full_cmd.status().expect("lnk cmd failed:");
+
+        (false, String::new())
+    }
+}
+
+fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
+    let log_name = format!("linkd_{}.log", &timestamp());
+    let log_file = File::create(&log_name).unwrap();
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/linkd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("linkd")
+        .output()
+        .expect("cargo build linkd failed");
+
+    let child = Command::new(&exec_path)
+        .env("RUST_BACKTRACE", "1")
+        .arg("--lnk-home")
+        .arg(lnk_home)
+        .arg("--track")
+        .arg("everything")
+        .arg("--protocol-listen")
+        .arg("127.0.0.1:8799")
+        .stdout(Stdio::from(log_file))
+        .spawn()
+        .expect("linkd failed to start");
+    println!("linkd stdout redirected to {}", &log_name);
+    thread::sleep(Duration::from_secs(1));
+    child
+}
+
+fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) -> bool {
+    let port = "9987";
+    let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
+    let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk-gitd")
+        .stdout(Stdio::inherit())
+        .output()
+        .expect("cargo build lnk-gitd failed");
+
+    let fork = Fork::from_ptmx().unwrap();
+
+    if let Some(_parent) = fork.is_parent().ok() {
+        println!("started lnk-gitd");
+        true
+    } else {
+        Command::new("systemd-socket-activate")
+            .arg("-l")
+            .arg(port)
+            .arg("--fdname=ssh")
+            .arg("-E")
+            .arg("SSH_AUTH_SOCK")
+            .arg("-E")
+            .arg("RUST_BACKTRACE")
+            .arg(&exec_path)
+            .arg(lnk_home)
+            .arg("--linkd-rpc-socket")
+            .arg(rpc_socket)
+            .arg("--push-seeds")
+            .arg("--fetch-seeds")
+            .arg("--linger-timeout")
+            .arg("10000")
+            .output()
+            .expect("lnk-gitd failed to start");
+        false
+    }
+}
+
+/// Returns true if this is the parent process,
+/// returns false if this is the child process.
+fn run_git_push() -> bool {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        let yes = b"yes\n";
+        let buf_reader = BufReader::new(parent);
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("git-push: {}", line);
+            if line.find("key fingerprint").is_some() {
+                parent.write_all(yes).unwrap();
+            }
+        }
+
+        true
+    } else {
+        Command::new("git")
+            .arg("push")
+            .arg("linkd")
+            .status()
+            .expect("failed to do git push");
+        false
+    }
+}
+
+/// Returns UNIX_TIME in millis.
+fn timestamp() -> u128 {
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap();
+    now.as_millis()
+}
+
+/// Returns the full path of `bins` manifest file.
+fn manifest_path() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/Cargo.toml", package_dir.strip_suffix("/tests").unwrap())
+}
+
+/// Returns the full path of `bins/target`.
+fn bins_target_dir() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/target", package_dir.strip_suffix("/tests").unwrap())
+}
+
+fn clean_up_known_hosts() {
+    // ssh-keygen -f "/home/pi/.ssh/known_hosts" -R "[127.0.0.1]:9987"
+    let home_dir = env!("HOME");
+    let known_hosts = format!("{}/.ssh/known_hosts", &home_dir);
+    let output = Command::new("ssh-keygen")
+        .arg("-f")
+        .arg(known_hosts)
+        .arg("-R")
+        .arg("[127.0.0.1]:9987")
+        .output()
+        .expect("failed to do ssh-keygen");
+    println!("ssh-keygen: {:?}", &output);
+}
diff --git a/link-git/src/protocol/fetch.rs b/link-git/src/protocol/fetch.rs
index a5a73065..562e433d 100644
--- a/link-git/src/protocol/fetch.rs
+++ b/link-git/src/protocol/fetch.rs
@@ -39,7 +39,7 @@ use super::{packwriter::PackWriter, remote_git_version, transport};
 // cf. https://lore.kernel.org/git/CD2XNXHACAXS.13J6JTWZPO1JA@schmidt/
 // Fixed in `git.git` 1ab13eb, which should land in 2.34
 fn must_namespace_want_ref(caps: &client::Capabilities) -> bool {
-    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.33.0").unwrap());
+    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version <= *FIXED_AFTER)
diff --git a/link-git/src/protocol/ls.rs b/link-git/src/protocol/ls.rs
index b3516454..9dafa538 100644
--- a/link-git/src/protocol/ls.rs
+++ b/link-git/src/protocol/ls.rs
@@ -24,7 +24,7 @@ use super::{remote_git_version, transport};
 // cf. https://lore.kernel.org/git/pMV5dJabxOBTD8kJBaPuWK0aS6OJhRQ7YFGwfhPCeSJEbPDrIFBza36nXBCgUCeUJWGmpjPI1rlOGvZJEh71Ruz4SqljndUwOCoBUDRHRDU=@eagain.st/
 fn must_namespace(caps: &client::Capabilities) -> bool {
     static MIN_GIT_VERSION_NAMESPACES: Lazy<Version> =
-        Lazy::new(|| Version::new("2.31.0").unwrap());
+        Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version < *MIN_GIT_VERSION_NAMESPACES)
-- 
2.32.0 (Apple Git-132)

Re: [PATCH radicle-link v2 1/1] Add integration test for bins

Alex Good <alex@memoryandthought.me>

Details

Message ID: <CAH_DpYR__4_vztrF2ebkw5XEdHC_SgaQV2S8ujZTL6cfT6xORQ@mail.gmail.com>
In-Reply-To: <20220817072940.74222-2-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On 17/08/22 12:29am, Han Xu wrote:
> diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
> new file mode 100644
> index 00000000..d6d3b4ae
> --- /dev/null
> +++ b/bins/tests/integration_test.rs

[...]

> +enum LnkCmd {
> +    ProfileCreate,
> +    ProfileGet,
> +    ProfilePeer,
> +    ProfileSshAdd,
> +    IdPersonCreate(String),  // the associated string is "the person's name".
> +    IdLocalSet(String),      // the associated string is "urn".
> +    IdProjectCreate(String), // the associated string is "the project name".
> +    Clone(String, String),   // the associated string is "urn", "peer_id"
> +}
> +
> +/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
> +/// Return.0: true if this is the parent (i.e. test) process,
> +///           false if this is the child (i.e. lnk) process.
> +/// Return.1: an output that depends on the `cmd`.
> +fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {

What's the reasoning behing using this API where we fork and check if
we're in a child vs using the std::process::Command APIs?

> diff --git a/link-git/src/protocol/fetch.rs b/link-git/src/protocol/fetch.rs
> index a5a73065..562e433d 100644
> --- a/link-git/src/protocol/fetch.rs
> +++ b/link-git/src/protocol/fetch.rs
> @@ -39,7 +39,7 @@ use super::{packwriter::PackWriter, remote_git_version, transport};
>  // cf. https://lore.kernel.org/git/CD2XNXHACAXS.13J6JTWZPO1JA@schmidt/
>  // Fixed in `git.git` 1ab13eb, which should land in 2.34
>  fn must_namespace_want_ref(caps: &client::Capabilities) -> bool {
> -    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.33.0").unwrap());
> +    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.25.0").unwrap());

This is quite surprising to me but as long as we've tested it and it
works that's great! We should probably change the comment above about it
landing in 2.34 though.

Re: [PATCH radicle-link v2 1/1] Add integration test for bins

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqf1ULSvJP4cEod8VjoT4NRbXHrGtkntC9nU7auotTGFVw@mail.gmail.com>
In-Reply-To: <CAH_DpYR__4_vztrF2ebkw5XEdHC_SgaQV2S8ujZTL6cfT6xORQ@mail.gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Wed, Aug 17, 2022 at 9:32 AM Alex Good <alex@memoryandthought.me> wrote:
>
> On 17/08/22 12:29am, Han Xu wrote:
> > diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
> > new file mode 100644
> > index 00000000..d6d3b4ae
> > --- /dev/null
> > +++ b/bins/tests/integration_test.rs
>
> [...]

Does this [...] mean something? (sorry I don't know)

>
> > +enum LnkCmd {
> > +    ProfileCreate,
> > +    ProfileGet,
> > +    ProfilePeer,
> > +    ProfileSshAdd,
> > +    IdPersonCreate(String),  // the associated string is "the person's name".
> > +    IdLocalSet(String),      // the associated string is "urn".
> > +    IdProjectCreate(String), // the associated string is "the project name".
> > +    Clone(String, String),   // the associated string is "urn", "peer_id"
> > +}
> > +
> > +/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
> > +/// Return.0: true if this is the parent (i.e. test) process,
> > +///           false if this is the child (i.e. lnk) process.
> > +/// Return.1: an output that depends on the `cmd`.
> > +fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
>
> What's the reasoning behing using this API where we fork and check if
> we're in a child vs using the std::process::Command APIs?

This is the most reliable way I could find to input a passphrase into
the lnk command's terminal input (TTY).

>
> > diff --git a/link-git/src/protocol/fetch.rs b/link-git/src/protocol/fetch.rs
> > index a5a73065..562e433d 100644
> > --- a/link-git/src/protocol/fetch.rs
> > +++ b/link-git/src/protocol/fetch.rs
> > @@ -39,7 +39,7 @@ use super::{packwriter::PackWriter, remote_git_version, transport};
> >  // cf. https://lore.kernel.org/git/CD2XNXHACAXS.13J6JTWZPO1JA@schmidt/
> >  // Fixed in `git.git` 1ab13eb, which should land in 2.34
> >  fn must_namespace_want_ref(caps: &client::Capabilities) -> bool {
> > -    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.33.0").unwrap());
> > +    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.25.0").unwrap());
>
> This is quite surprising to me but as long as we've tested it and it
> works that's great! We should probably change the comment above about it
> landing in 2.34 though.

It was surprising to me too. It seems the fix 1ab13eb did land to git 2.34:

https://github.com/git/git/blob/master/Documentation/RelNotes/2.34.0.txt#L248

But I'm not sure if it impacts the exact code path for our case. I
will update the comment about the reason to change to 2.25.0 based on
testing.

[PATCH radicle-link v3 0/1] Add integration test for bins and fix git version checks

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220818225452.26321-1-keepsimple@gmail.com>
In-Reply-To: <20220817072940.74222-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

This is the 3rd version (re-rolling) patch to add an integration test for `bins` package.

The test case scenario follows the steps in https://github.com/alexjg/linkd-playground ,
except it is in Rust now and we can just run `cargo test`. To see a more helpful output,
please run `cargo test -- --nocapture` from the `bins/tests` directory.

The only non-test code change is in git version checks. In our testing, it shows that
git 2.25.1 on Ubuntu 20.04 and git 2.30.2 on Debian 11 do not need the workaround that
adds "namespaces" manually. Hence fixing the git version checked against.

This patch also addresses the comments from the 2nd version, and I opted to re-rolling
instead of adding new commits.

Han Xu (1):
  Add integration test for bins and fix git version checks

 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 449 +++++++++++++++++++++++++++++++++
 link-git/src/protocol/fetch.rs |   5 +-
 link-git/src/protocol/ls.rs    |   5 +-
 6 files changed, 500 insertions(+), 2 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v3 1/1] Add integration test for bins and fix git version checks

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220818225452.26321-2-keepsimple@gmail.com>
In-Reply-To: <20220818225452.26321-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +500 -2

---
 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 449 +++++++++++++++++++++++++++++++++
 link-git/src/protocol/fetch.rs |   5 +-
 link-git/src/protocol/ls.rs    |   5 +-
 6 files changed, 500 insertions(+), 2 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

diff --git a/bins/Cargo.lock b/bins/Cargo.lock
index d5756144..4d0ea413 100644
--- a/bins/Cargo.lock
@@ -838,6 +838,17 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "errno"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e2b2decb0484e15560df3210cf0d78654bb0864b2c138977c07e377a1bae0e2"
+dependencies = [
+ "kernel32-sys",
+ "libc",
+ "winapi 0.2.8",
+]
+
 [[package]]
 name = "event-listener"
 version = "2.5.2"
@@ -2945,6 +2956,16 @@ dependencies = [
  "human_format",
 ]
 
+[[package]]
+name = "pty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50f3d255966981eb4e4c5df3e983e6f7d163221f547406d83b6a460ff5c5ee8"
+dependencies = [
+ "errno",
+ "libc",
+]
+
 [[package]]
 name = "quanta"
 version = "0.4.1"
@@ -3567,6 +3588,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "tests"
+version = "0.1.0"
+dependencies = [
+ "pty",
+ "serde_json",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
diff --git a/bins/Cargo.toml b/bins/Cargo.toml
index 3d1786f4..99b3c69f 100644
--- a/bins/Cargo.toml
@@ -5,4 +5,5 @@ members = [
   "lnk-gitd",
   "lnk-identities-dev",
   "lnk-profile-dev",
+  "tests",
 ]
diff --git a/bins/tests/Cargo.toml b/bins/tests/Cargo.toml
new file mode 100644
index 00000000..3ae82769
--- /dev/null
@@ -0,0 +1,13 @@
+[package]
+name = "tests"
+version = "0.1.0"
+authors = ["Han Xu <keepsimple@gmail.com>"]
+edition = "2021"
+
+[dev-dependencies]
+pty = "0.2"
+serde_json = "1.0"
+
+[[test]]
+name = "integration_test"
+path = "integration_test.rs"
diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
new file mode 100644
index 00000000..770f1b5e
--- /dev/null
@@ -0,0 +1,449 @@
+// Copyright © 2022 The Radicle Link Contributors
+//
+// This file is part of radicle-link, distributed under the GPLv3 with Radicle
+// Linking Exception. For full terms see the included LICENSE file.
+
+//! Integration test to exercise the programs in `bins`.
+
+use pty::fork::*;
+use serde_json::{json, Value};
+use std::{
+    env,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    process::{Child, Command, Stdio},
+    thread,
+    time::{Duration, SystemTime},
+};
+
+/// This test is inspired by https://github.com/alexjg/linkd-playground
+///
+/// Tests a typical scenario: there are two peer nodes and one seed node.
+/// The main steps are:
+///   - Setup a profile for each node in tmp home directories.
+///   - Setup SSH keys for each profile.
+///   - Create identities.
+///   - Create a local repo for peer 1.
+///   - Start `linkd` for the seed, and `lnk-gitd` for peer 1.
+///   - Push peer 1 repo to its monorepo and to the seed.
+///   - Clone the peer 1 repo to peer 2 via seed.
+#[test]
+fn two_peers_and_a_seed() {
+    let peer1_home = "/tmp/link-local-1";
+    let peer2_home = "/tmp/link-local-2";
+    let seed_home = "/tmp/seed-home";
+    let passphrase = b"play\n";
+
+    println!("\n== create lnk homes for two peers and one seed ==\n");
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Creating local link 1 identity ==\n");
+    let peer1_name = "sockpuppet1".to_string();
+    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer1_name), peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let urn1 = v["urn"].as_str().unwrap().to_string();
+    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn1), peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Creating local link 2 identity ==\n");
+    let peer2_name = "sockpuppet2".to_string();
+    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer2_name), peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let urn2 = v["urn"].as_str().unwrap().to_string();
+    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn2), peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Create a local repository ==\n");
+    let peer1_proj = format!("peer1_proj_{}", timestamp());
+    let (is_parent, output) = run_lnk(
+        LnkCmd::IdProjectCreate(peer1_proj.clone()),
+        peer1_home,
+        passphrase,
+    );
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let proj_urn = v["urn"].as_str().unwrap().to_string();
+    println!("our project URN: {}", &proj_urn);
+
+    println!("\n== Add the seed to the local peer seed configs ==\n");
+    let (is_parent, seed_peer_id) = run_lnk(LnkCmd::ProfilePeer, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let seed_endpoint = format!("{}@127.0.0.1:8799", &seed_peer_id);
+
+    let (is_parent, peer1_profile) = run_lnk(LnkCmd::ProfileGet, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
+    let mut peer1_f = File::create(peer1_seed).unwrap();
+    peer1_f.write_all(seed_endpoint.as_bytes()).unwrap();
+
+    let (is_parent, peer2_profile) = run_lnk(LnkCmd::ProfileGet, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
+    let mut peer2_f = File::create(peer2_seed).unwrap();
+    peer2_f.write_all(seed_endpoint.as_bytes()).unwrap();
+
+    println!("\n== Start the seed linkd ==\n");
+    let manifest_path = manifest_path();
+    let mut linkd = spawn_linkd(seed_home, &manifest_path);
+
+    println!("\n== Start the peer 1 gitd ==\n");
+    let (is_parent, peer1_peer_id) = run_lnk(LnkCmd::ProfilePeer, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    spawn_lnk_gitd(peer1_home, &manifest_path, &peer1_peer_id);
+
+    println!("\n== Make some changes in the repo ==\n");
+    env::set_current_dir(&peer1_proj).unwrap();
+    let mut test_file = File::create("test").unwrap();
+    test_file.write_all(b"test").unwrap();
+    Command::new("git")
+        .arg("add")
+        .arg("test")
+        .output()
+        .expect("failed to do git add");
+    let output = Command::new("git")
+        .arg("commit")
+        .arg("-m")
+        .arg("test commit")
+        .output()
+        .expect("failed to do git commit");
+    println!("git-commit: {:?}", &output);
+
+    println!("\n== Add the linkd remote to the repo ==\n");
+    let remote_url = format!("ssh://rad@127.0.0.1:9987/{}.git", &proj_urn);
+    Command::new("git")
+        .arg("remote")
+        .arg("add")
+        .arg("linkd")
+        .arg(remote_url)
+        .output()
+        .expect("failed to do git remote add");
+
+    clean_up_known_hosts();
+
+    let is_parent = run_git_push();
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Clone to peer2 ==\n");
+
+    env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
+    let (is_parent, _) = run_lnk(
+        LnkCmd::Clone(proj_urn, peer1_peer_id),
+        peer2_home,
+        passphrase,
+    );
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Kill linkd (seed) ==\n");
+
+    linkd.kill().ok();
+}
+
+enum LnkCmd {
+    ProfileCreate,
+    ProfileGet,
+    ProfilePeer,
+    ProfileSshAdd,
+    IdPersonCreate(String),  // the associated string is "the person's name".
+    IdLocalSet(String),      // the associated string is "urn".
+    IdProjectCreate(String), // the associated string is "the project name".
+    Clone(String, String),   // the associated string is "urn", "peer_id"
+}
+
+/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
+/// Return.0: true if this is the parent (i.e. test) process,
+///           false if this is the child (i.e. lnk) process.
+/// Return.1: an output that depends on the `cmd`.
+fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        // Input the passphrase if necessary.
+        match cmd {
+            LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
+                parent.write_all(passphrase).unwrap();
+                println!("{}: wrote passphase", lnk_home);
+            },
+            _ => {},
+        }
+
+        // Print the output and decode them if necessary.
+        let buf_reader = BufReader::new(parent);
+        let mut output = String::new();
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("{}: {}", lnk_home, line);
+
+            match cmd {
+                LnkCmd::IdPersonCreate(ref _name) => {
+                    if line.find("\"urn\":").is_some() {
+                        output = line; // get the line with URN.
+                    }
+                },
+                LnkCmd::IdProjectCreate(ref _name) => {
+                    if line.find("\"urn\":").is_some() {
+                        output = line; // get the line with URN.
+                    }
+                },
+                LnkCmd::ProfileGet => {
+                    output = line; // get the last line for profile id.
+                },
+                LnkCmd::ProfilePeer => {
+                    output = line; // get the last line for peer id.
+                },
+                LnkCmd::Clone(ref _urn, ref _peer) => {
+                    output = line;
+                },
+                _ => {},
+            }
+        }
+
+        (true, output)
+    } else {
+        // Child process is to run `lnk`.
+        let manifest_path = manifest_path();
+
+        // cargo run \
+        // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
+        // -p lnk -- "$@"
+        let mut lnk_cmd = Command::new("cargo");
+        lnk_cmd
+            .env("LNK_HOME", lnk_home)
+            .arg("run")
+            .arg("--manifest-path")
+            .arg(manifest_path)
+            .arg("-p")
+            .arg("lnk")
+            .arg("--");
+        let full_cmd = match cmd {
+            LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
+            LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
+            LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
+            LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
+            LnkCmd::IdPersonCreate(name) => {
+                let payload = json!({ "name": name });
+                lnk_cmd
+                    .arg("identities")
+                    .arg("person")
+                    .arg("create")
+                    .arg("new")
+                    .arg("--payload")
+                    .arg(payload.to_string())
+            },
+            LnkCmd::IdLocalSet(urn) => lnk_cmd
+                .arg("identities")
+                .arg("local")
+                .arg("set")
+                .arg("--urn")
+                .arg(urn),
+            LnkCmd::IdProjectCreate(name) => {
+                let payload = json!({"name": name, "default_branch": "master"});
+                let project_path = format!("./{}", name);
+                lnk_cmd
+                    .arg("identities")
+                    .arg("project")
+                    .arg("create")
+                    .arg("new")
+                    .arg("--path")
+                    .arg(project_path)
+                    .arg("--payload")
+                    .arg(payload.to_string())
+            },
+            LnkCmd::Clone(urn, peer_id) => {
+                let peer2_proj = format!("peer2_proj_{}", timestamp());
+                lnk_cmd
+                    .arg("clone")
+                    .arg("--urn")
+                    .arg(urn)
+                    .arg("--path")
+                    .arg(peer2_proj)
+                    .arg("--peer")
+                    .arg(peer_id)
+            },
+        };
+        full_cmd.status().expect("lnk cmd failed:");
+
+        (false, String::new())
+    }
+}
+
+fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
+    let log_name = format!("linkd_{}.log", &timestamp());
+    let log_file = File::create(&log_name).unwrap();
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/linkd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("linkd")
+        .output()
+        .expect("cargo build linkd failed");
+
+    let child = Command::new(&exec_path)
+        .env("RUST_BACKTRACE", "1")
+        .arg("--lnk-home")
+        .arg(lnk_home)
+        .arg("--track")
+        .arg("everything")
+        .arg("--protocol-listen")
+        .arg("127.0.0.1:8799")
+        .stdout(Stdio::from(log_file))
+        .spawn()
+        .expect("linkd failed to start");
+    println!("linkd stdout redirected to {}", &log_name);
+    thread::sleep(Duration::from_secs(1));
+    child
+}
+
+fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
+    let port = "9987";
+    let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
+    let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk-gitd")
+        .stdout(Stdio::inherit())
+        .output()
+        .expect("cargo build lnk-gitd failed");
+
+        Command::new("systemd-socket-activate")
+            .arg("-l")
+            .arg(port)
+            .arg("--fdname=ssh")
+            .arg("-E")
+            .arg("SSH_AUTH_SOCK")
+            .arg("-E")
+            .arg("RUST_BACKTRACE")
+            .arg(&exec_path)
+            .arg(lnk_home)
+            .arg("--linkd-rpc-socket")
+            .arg(rpc_socket)
+            .arg("--push-seeds")
+            .arg("--fetch-seeds")
+            .arg("--linger-timeout")
+            .arg("10000")
+            .spawn()
+            .expect("lnk-gitd failed to start");
+            println!("started lnk-gitd");
+
+}
+
+/// Returns true if this is the parent process,
+/// returns false if this is the child process.
+fn run_git_push() -> bool {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        let yes = b"yes\n";
+        let buf_reader = BufReader::new(parent);
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("git-push: {}", line);
+            if line.find("key fingerprint").is_some() {
+                parent.write_all(yes).unwrap();
+            }
+        }
+
+        true
+    } else {
+        Command::new("git")
+            .arg("push")
+            .arg("linkd")
+            .status()
+            .expect("failed to do git push");
+        false
+    }
+}
+
+/// Returns UNIX_TIME in millis.
+fn timestamp() -> u128 {
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap();
+    now.as_millis()
+}
+
+/// Returns the full path of `bins` manifest file.
+fn manifest_path() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/Cargo.toml", package_dir.strip_suffix("/tests").unwrap())
+}
+
+/// Returns the full path of `bins/target`.
+fn bins_target_dir() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/target", package_dir.strip_suffix("/tests").unwrap())
+}
+
+fn clean_up_known_hosts() {
+    // ssh-keygen -f "/home/pi/.ssh/known_hosts" -R "[127.0.0.1]:9987"
+    let home_dir = env!("HOME");
+    let known_hosts = format!("{}/.ssh/known_hosts", &home_dir);
+    let output = Command::new("ssh-keygen")
+        .arg("-f")
+        .arg(known_hosts)
+        .arg("-R")
+        .arg("[127.0.0.1]:9987")
+        .output()
+        .expect("failed to do ssh-keygen");
+    println!("ssh-keygen: {:?}", &output);
+}
diff --git a/link-git/src/protocol/fetch.rs b/link-git/src/protocol/fetch.rs
index a5a73065..98b92b51 100644
--- a/link-git/src/protocol/fetch.rs
+++ b/link-git/src/protocol/fetch.rs
@@ -38,8 +38,11 @@ use super::{packwriter::PackWriter, remote_git_version, transport};
 //
 // cf. https://lore.kernel.org/git/CD2XNXHACAXS.13J6JTWZPO1JA@schmidt/
 // Fixed in `git.git` 1ab13eb, which should land in 2.34
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace_want_ref(caps: &client::Capabilities) -> bool {
-    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.33.0").unwrap());
+    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version <= *FIXED_AFTER)
diff --git a/link-git/src/protocol/ls.rs b/link-git/src/protocol/ls.rs
index b3516454..de95b4a3 100644
--- a/link-git/src/protocol/ls.rs
+++ b/link-git/src/protocol/ls.rs
@@ -22,9 +22,12 @@ use super::{remote_git_version, transport};
 // Work around `git-upload-pack` not handling namespaces properly
 //
 // cf. https://lore.kernel.org/git/pMV5dJabxOBTD8kJBaPuWK0aS6OJhRQ7YFGwfhPCeSJEbPDrIFBza36nXBCgUCeUJWGmpjPI1rlOGvZJEh71Ruz4SqljndUwOCoBUDRHRDU=@eagain.st/
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace(caps: &client::Capabilities) -> bool {
     static MIN_GIT_VERSION_NAMESPACES: Lazy<Version> =
-        Lazy::new(|| Version::new("2.31.0").unwrap());
+        Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version < *MIN_GIT_VERSION_NAMESPACES)
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v4 0/3] add integration test to bins package

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220825054156.57232-1-keepsimple@gmail.com>
In-Reply-To: <20220818225452.26321-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

This patch is an update on the previous v3 patch, with the following changes:

- Added a check for refs/heads/master between peers. This is because I noticed
a bug where `git push` from peer1 only pushs an older commit as `heads/master`
to the seed node.

- Found the root cause of the above bug and included a fix in 
cli/gitd-lib/src/hooks.rs. The root cause is that currently a peer does 
request_pull before updating the signed refs, hence the request pull only
has an old signed refs for heads/master.

Han Xu (3):
  Add integration test for bins and fix git version checks
  check peer1 and peer2 ref/heads/master
  fix post_receive to send updated refs in requst_pull

 bins/Cargo.lock                |  29 ++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 468 +++++++++++++++++++++++++++++++++
 cli/gitd-lib/src/hooks.rs      |  19 +-
 link-git/src/protocol/fetch.rs |   5 +-
 link-git/src/protocol/ls.rs    |   5 +-
 7 files changed, 530 insertions(+), 10 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v4 1/3] Add integration test for bins and fix git version checks

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220825054156.57232-2-keepsimple@gmail.com>
In-Reply-To: <20220825054156.57232-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +500 -2

---
 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 449 +++++++++++++++++++++++++++++++++
 link-git/src/protocol/fetch.rs |   5 +-
 link-git/src/protocol/ls.rs    |   5 +-
 6 files changed, 500 insertions(+), 2 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

diff --git a/bins/Cargo.lock b/bins/Cargo.lock
index d5756144..4d0ea413 100644
--- a/bins/Cargo.lock
@@ -838,6 +838,17 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "errno"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e2b2decb0484e15560df3210cf0d78654bb0864b2c138977c07e377a1bae0e2"
+dependencies = [
+ "kernel32-sys",
+ "libc",
+ "winapi 0.2.8",
+]
+
 [[package]]
 name = "event-listener"
 version = "2.5.2"
@@ -2945,6 +2956,16 @@ dependencies = [
  "human_format",
 ]
 
+[[package]]
+name = "pty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50f3d255966981eb4e4c5df3e983e6f7d163221f547406d83b6a460ff5c5ee8"
+dependencies = [
+ "errno",
+ "libc",
+]
+
 [[package]]
 name = "quanta"
 version = "0.4.1"
@@ -3567,6 +3588,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "tests"
+version = "0.1.0"
+dependencies = [
+ "pty",
+ "serde_json",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
diff --git a/bins/Cargo.toml b/bins/Cargo.toml
index 3d1786f4..99b3c69f 100644
--- a/bins/Cargo.toml
@@ -5,4 +5,5 @@ members = [
   "lnk-gitd",
   "lnk-identities-dev",
   "lnk-profile-dev",
+  "tests",
 ]
diff --git a/bins/tests/Cargo.toml b/bins/tests/Cargo.toml
new file mode 100644
index 00000000..3ae82769
--- /dev/null
@@ -0,0 +1,13 @@
+[package]
+name = "tests"
+version = "0.1.0"
+authors = ["Han Xu <keepsimple@gmail.com>"]
+edition = "2021"
+
+[dev-dependencies]
+pty = "0.2"
+serde_json = "1.0"
+
+[[test]]
+name = "integration_test"
+path = "integration_test.rs"
diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
new file mode 100644
index 00000000..770f1b5e
--- /dev/null
@@ -0,0 +1,449 @@
+// Copyright © 2022 The Radicle Link Contributors
+//
+// This file is part of radicle-link, distributed under the GPLv3 with Radicle
+// Linking Exception. For full terms see the included LICENSE file.
+
+//! Integration test to exercise the programs in `bins`.
+
+use pty::fork::*;
+use serde_json::{json, Value};
+use std::{
+    env,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    process::{Child, Command, Stdio},
+    thread,
+    time::{Duration, SystemTime},
+};
+
+/// This test is inspired by https://github.com/alexjg/linkd-playground
+///
+/// Tests a typical scenario: there are two peer nodes and one seed node.
+/// The main steps are:
+///   - Setup a profile for each node in tmp home directories.
+///   - Setup SSH keys for each profile.
+///   - Create identities.
+///   - Create a local repo for peer 1.
+///   - Start `linkd` for the seed, and `lnk-gitd` for peer 1.
+///   - Push peer 1 repo to its monorepo and to the seed.
+///   - Clone the peer 1 repo to peer 2 via seed.
+#[test]
+fn two_peers_and_a_seed() {
+    let peer1_home = "/tmp/link-local-1";
+    let peer2_home = "/tmp/link-local-2";
+    let seed_home = "/tmp/seed-home";
+    let passphrase = b"play\n";
+
+    println!("\n== create lnk homes for two peers and one seed ==\n");
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Creating local link 1 identity ==\n");
+    let peer1_name = "sockpuppet1".to_string();
+    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer1_name), peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let urn1 = v["urn"].as_str().unwrap().to_string();
+    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn1), peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Creating local link 2 identity ==\n");
+    let peer2_name = "sockpuppet2".to_string();
+    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer2_name), peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let urn2 = v["urn"].as_str().unwrap().to_string();
+    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn2), peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Create a local repository ==\n");
+    let peer1_proj = format!("peer1_proj_{}", timestamp());
+    let (is_parent, output) = run_lnk(
+        LnkCmd::IdProjectCreate(peer1_proj.clone()),
+        peer1_home,
+        passphrase,
+    );
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let proj_urn = v["urn"].as_str().unwrap().to_string();
+    println!("our project URN: {}", &proj_urn);
+
+    println!("\n== Add the seed to the local peer seed configs ==\n");
+    let (is_parent, seed_peer_id) = run_lnk(LnkCmd::ProfilePeer, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let seed_endpoint = format!("{}@127.0.0.1:8799", &seed_peer_id);
+
+    let (is_parent, peer1_profile) = run_lnk(LnkCmd::ProfileGet, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
+    let mut peer1_f = File::create(peer1_seed).unwrap();
+    peer1_f.write_all(seed_endpoint.as_bytes()).unwrap();
+
+    let (is_parent, peer2_profile) = run_lnk(LnkCmd::ProfileGet, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
+    let mut peer2_f = File::create(peer2_seed).unwrap();
+    peer2_f.write_all(seed_endpoint.as_bytes()).unwrap();
+
+    println!("\n== Start the seed linkd ==\n");
+    let manifest_path = manifest_path();
+    let mut linkd = spawn_linkd(seed_home, &manifest_path);
+
+    println!("\n== Start the peer 1 gitd ==\n");
+    let (is_parent, peer1_peer_id) = run_lnk(LnkCmd::ProfilePeer, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    spawn_lnk_gitd(peer1_home, &manifest_path, &peer1_peer_id);
+
+    println!("\n== Make some changes in the repo ==\n");
+    env::set_current_dir(&peer1_proj).unwrap();
+    let mut test_file = File::create("test").unwrap();
+    test_file.write_all(b"test").unwrap();
+    Command::new("git")
+        .arg("add")
+        .arg("test")
+        .output()
+        .expect("failed to do git add");
+    let output = Command::new("git")
+        .arg("commit")
+        .arg("-m")
+        .arg("test commit")
+        .output()
+        .expect("failed to do git commit");
+    println!("git-commit: {:?}", &output);
+
+    println!("\n== Add the linkd remote to the repo ==\n");
+    let remote_url = format!("ssh://rad@127.0.0.1:9987/{}.git", &proj_urn);
+    Command::new("git")
+        .arg("remote")
+        .arg("add")
+        .arg("linkd")
+        .arg(remote_url)
+        .output()
+        .expect("failed to do git remote add");
+
+    clean_up_known_hosts();
+
+    let is_parent = run_git_push();
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Clone to peer2 ==\n");
+
+    env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
+    let (is_parent, _) = run_lnk(
+        LnkCmd::Clone(proj_urn, peer1_peer_id),
+        peer2_home,
+        passphrase,
+    );
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Kill linkd (seed) ==\n");
+
+    linkd.kill().ok();
+}
+
+enum LnkCmd {
+    ProfileCreate,
+    ProfileGet,
+    ProfilePeer,
+    ProfileSshAdd,
+    IdPersonCreate(String),  // the associated string is "the person's name".
+    IdLocalSet(String),      // the associated string is "urn".
+    IdProjectCreate(String), // the associated string is "the project name".
+    Clone(String, String),   // the associated string is "urn", "peer_id"
+}
+
+/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
+/// Return.0: true if this is the parent (i.e. test) process,
+///           false if this is the child (i.e. lnk) process.
+/// Return.1: an output that depends on the `cmd`.
+fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        // Input the passphrase if necessary.
+        match cmd {
+            LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
+                parent.write_all(passphrase).unwrap();
+                println!("{}: wrote passphase", lnk_home);
+            },
+            _ => {},
+        }
+
+        // Print the output and decode them if necessary.
+        let buf_reader = BufReader::new(parent);
+        let mut output = String::new();
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("{}: {}", lnk_home, line);
+
+            match cmd {
+                LnkCmd::IdPersonCreate(ref _name) => {
+                    if line.find("\"urn\":").is_some() {
+                        output = line; // get the line with URN.
+                    }
+                },
+                LnkCmd::IdProjectCreate(ref _name) => {
+                    if line.find("\"urn\":").is_some() {
+                        output = line; // get the line with URN.
+                    }
+                },
+                LnkCmd::ProfileGet => {
+                    output = line; // get the last line for profile id.
+                },
+                LnkCmd::ProfilePeer => {
+                    output = line; // get the last line for peer id.
+                },
+                LnkCmd::Clone(ref _urn, ref _peer) => {
+                    output = line;
+                },
+                _ => {},
+            }
+        }
+
+        (true, output)
+    } else {
+        // Child process is to run `lnk`.
+        let manifest_path = manifest_path();
+
+        // cargo run \
+        // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
+        // -p lnk -- "$@"
+        let mut lnk_cmd = Command::new("cargo");
+        lnk_cmd
+            .env("LNK_HOME", lnk_home)
+            .arg("run")
+            .arg("--manifest-path")
+            .arg(manifest_path)
+            .arg("-p")
+            .arg("lnk")
+            .arg("--");
+        let full_cmd = match cmd {
+            LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
+            LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
+            LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
+            LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
+            LnkCmd::IdPersonCreate(name) => {
+                let payload = json!({ "name": name });
+                lnk_cmd
+                    .arg("identities")
+                    .arg("person")
+                    .arg("create")
+                    .arg("new")
+                    .arg("--payload")
+                    .arg(payload.to_string())
+            },
+            LnkCmd::IdLocalSet(urn) => lnk_cmd
+                .arg("identities")
+                .arg("local")
+                .arg("set")
+                .arg("--urn")
+                .arg(urn),
+            LnkCmd::IdProjectCreate(name) => {
+                let payload = json!({"name": name, "default_branch": "master"});
+                let project_path = format!("./{}", name);
+                lnk_cmd
+                    .arg("identities")
+                    .arg("project")
+                    .arg("create")
+                    .arg("new")
+                    .arg("--path")
+                    .arg(project_path)
+                    .arg("--payload")
+                    .arg(payload.to_string())
+            },
+            LnkCmd::Clone(urn, peer_id) => {
+                let peer2_proj = format!("peer2_proj_{}", timestamp());
+                lnk_cmd
+                    .arg("clone")
+                    .arg("--urn")
+                    .arg(urn)
+                    .arg("--path")
+                    .arg(peer2_proj)
+                    .arg("--peer")
+                    .arg(peer_id)
+            },
+        };
+        full_cmd.status().expect("lnk cmd failed:");
+
+        (false, String::new())
+    }
+}
+
+fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
+    let log_name = format!("linkd_{}.log", &timestamp());
+    let log_file = File::create(&log_name).unwrap();
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/linkd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("linkd")
+        .output()
+        .expect("cargo build linkd failed");
+
+    let child = Command::new(&exec_path)
+        .env("RUST_BACKTRACE", "1")
+        .arg("--lnk-home")
+        .arg(lnk_home)
+        .arg("--track")
+        .arg("everything")
+        .arg("--protocol-listen")
+        .arg("127.0.0.1:8799")
+        .stdout(Stdio::from(log_file))
+        .spawn()
+        .expect("linkd failed to start");
+    println!("linkd stdout redirected to {}", &log_name);
+    thread::sleep(Duration::from_secs(1));
+    child
+}
+
+fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
+    let port = "9987";
+    let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
+    let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk-gitd")
+        .stdout(Stdio::inherit())
+        .output()
+        .expect("cargo build lnk-gitd failed");
+
+        Command::new("systemd-socket-activate")
+            .arg("-l")
+            .arg(port)
+            .arg("--fdname=ssh")
+            .arg("-E")
+            .arg("SSH_AUTH_SOCK")
+            .arg("-E")
+            .arg("RUST_BACKTRACE")
+            .arg(&exec_path)
+            .arg(lnk_home)
+            .arg("--linkd-rpc-socket")
+            .arg(rpc_socket)
+            .arg("--push-seeds")
+            .arg("--fetch-seeds")
+            .arg("--linger-timeout")
+            .arg("10000")
+            .spawn()
+            .expect("lnk-gitd failed to start");
+            println!("started lnk-gitd");
+
+}
+
+/// Returns true if this is the parent process,
+/// returns false if this is the child process.
+fn run_git_push() -> bool {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        let yes = b"yes\n";
+        let buf_reader = BufReader::new(parent);
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("git-push: {}", line);
+            if line.find("key fingerprint").is_some() {
+                parent.write_all(yes).unwrap();
+            }
+        }
+
+        true
+    } else {
+        Command::new("git")
+            .arg("push")
+            .arg("linkd")
+            .status()
+            .expect("failed to do git push");
+        false
+    }
+}
+
+/// Returns UNIX_TIME in millis.
+fn timestamp() -> u128 {
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap();
+    now.as_millis()
+}
+
+/// Returns the full path of `bins` manifest file.
+fn manifest_path() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/Cargo.toml", package_dir.strip_suffix("/tests").unwrap())
+}
+
+/// Returns the full path of `bins/target`.
+fn bins_target_dir() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/target", package_dir.strip_suffix("/tests").unwrap())
+}
+
+fn clean_up_known_hosts() {
+    // ssh-keygen -f "/home/pi/.ssh/known_hosts" -R "[127.0.0.1]:9987"
+    let home_dir = env!("HOME");
+    let known_hosts = format!("{}/.ssh/known_hosts", &home_dir);
+    let output = Command::new("ssh-keygen")
+        .arg("-f")
+        .arg(known_hosts)
+        .arg("-R")
+        .arg("[127.0.0.1]:9987")
+        .output()
+        .expect("failed to do ssh-keygen");
+    println!("ssh-keygen: {:?}", &output);
+}
diff --git a/link-git/src/protocol/fetch.rs b/link-git/src/protocol/fetch.rs
index a5a73065..98b92b51 100644
--- a/link-git/src/protocol/fetch.rs
+++ b/link-git/src/protocol/fetch.rs
@@ -38,8 +38,11 @@ use super::{packwriter::PackWriter, remote_git_version, transport};
 //
 // cf. https://lore.kernel.org/git/CD2XNXHACAXS.13J6JTWZPO1JA@schmidt/
 // Fixed in `git.git` 1ab13eb, which should land in 2.34
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace_want_ref(caps: &client::Capabilities) -> bool {
-    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.33.0").unwrap());
+    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version <= *FIXED_AFTER)
diff --git a/link-git/src/protocol/ls.rs b/link-git/src/protocol/ls.rs
index b3516454..de95b4a3 100644
--- a/link-git/src/protocol/ls.rs
+++ b/link-git/src/protocol/ls.rs
@@ -22,9 +22,12 @@ use super::{remote_git_version, transport};
 // Work around `git-upload-pack` not handling namespaces properly
 //
 // cf. https://lore.kernel.org/git/pMV5dJabxOBTD8kJBaPuWK0aS6OJhRQ7YFGwfhPCeSJEbPDrIFBza36nXBCgUCeUJWGmpjPI1rlOGvZJEh71Ruz4SqljndUwOCoBUDRHRDU=@eagain.st/
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace(caps: &client::Capabilities) -> bool {
     static MIN_GIT_VERSION_NAMESPACES: Lazy<Version> =
-        Lazy::new(|| Version::new("2.31.0").unwrap());
+        Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version < *MIN_GIT_VERSION_NAMESPACES)
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v4 2/3] check peer1 and peer2 ref/heads/master

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220825054156.57232-3-keepsimple@gmail.com>
In-Reply-To: <20220825054156.57232-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +25 -6

---
 bins/tests/integration_test.rs | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
index 770f1b5e..ea6815ab 100644
--- a/bins/tests/integration_test.rs
@@ -170,11 +170,15 @@ fn two_peers_and_a_seed() {
         return;
     }
 
+    let peer1_last_commit = git_last_commit();
+    println!("\n== peer1 project last commit: {} ==\n", &peer1_last_commit);
+
     println!("\n== Clone to peer2 ==\n");
 
     env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
+    let peer2_proj = format!("peer2_proj_{}", timestamp());
     let (is_parent, _) = run_lnk(
-        LnkCmd::Clone(proj_urn, peer1_peer_id),
+        LnkCmd::Clone(proj_urn, peer1_peer_id, peer2_proj.clone()),
         peer2_home,
         passphrase,
     );
@@ -182,9 +186,16 @@ fn two_peers_and_a_seed() {
         return;
     }
 
-    println!("\n== Kill linkd (seed) ==\n");
+    env::set_current_dir(peer2_proj).unwrap();
+    let peer2_last_commit = git_last_commit();
+    println!("\n== peer1 proj last commit: {}", &peer1_last_commit);
+    println!("\n== peer2 proj last commit: {}", &peer2_last_commit);
+
+    println!("\n== Cleanup: kill linkd (seed) ==\n");
 
     linkd.kill().ok();
+
+    assert_eq!(peer1_last_commit, peer2_last_commit);
 }
 
 enum LnkCmd {
@@ -195,7 +206,7 @@ enum LnkCmd {
     IdPersonCreate(String),  // the associated string is "the person's name".
     IdLocalSet(String),      // the associated string is "urn".
     IdProjectCreate(String), // the associated string is "the project name".
-    Clone(String, String),   // the associated string is "urn", "peer_id"
+    Clone(String, String, String),   // the associated string is "urn", "peer_id", "path"
 }
 
 /// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
@@ -238,7 +249,7 @@ fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
                 LnkCmd::ProfilePeer => {
                     output = line; // get the last line for peer id.
                 },
-                LnkCmd::Clone(ref _urn, ref _peer) => {
+                LnkCmd::Clone(ref _urn, ref _peer, ref _path) => {
                     output = line;
                 },
                 _ => {},
@@ -296,8 +307,7 @@ fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
                     .arg("--payload")
                     .arg(payload.to_string())
             },
-            LnkCmd::Clone(urn, peer_id) => {
-                let peer2_proj = format!("peer2_proj_{}", timestamp());
+            LnkCmd::Clone(urn, peer_id, peer2_proj) => {
                 lnk_cmd
                     .arg("clone")
                     .arg("--urn")
@@ -414,6 +424,15 @@ fn run_git_push() -> bool {
     }
 }
 
+fn git_last_commit() -> String {
+    let output = Command::new("git")
+        .arg("rev-parse")
+        .arg("HEAD")
+        .output()
+        .expect("failed to run git rev-parse");
+    String::from_utf8_lossy(&output.stdout).to_string()
+}
+
 /// Returns UNIX_TIME in millis.
 fn timestamp() -> u128 {
     let now = SystemTime::now()
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v4 3/3] fix post_receive to send updated refs in requst_pull

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220825054156.57232-4-keepsimple@gmail.com>
In-Reply-To: <20220825054156.57232-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +11 -8

---
 cli/gitd-lib/src/hooks.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/cli/gitd-lib/src/hooks.rs b/cli/gitd-lib/src/hooks.rs
index 5e928ffd..cf37d71f 100644
--- a/cli/gitd-lib/src/hooks.rs
+++ b/cli/gitd-lib/src/hooks.rs
@@ -77,6 +77,16 @@ where
         E: std::error::Error + Send + 'static,
         P: ProgressReporter<Error = E>,
     {
+        // update the signed refs before a possible request_pull,
+        // so that the peer can receive the latest refs.
+        let at = update_signed_refs(
+            reporter,
+            self.spawner.clone(),
+            self.pool.clone(),
+            urn.clone(),
+        )
+        .await?;
+
         if self.post_receive.request_pull {
             tracing::info!("executing request-pull");
             request_pull(reporter, &self.client, &self.seeds, urn.clone()).await?;
@@ -87,14 +97,7 @@ where
             )
             .await?;
         }
-        let at = match update_signed_refs(
-            reporter,
-            self.spawner.clone(),
-            self.pool.clone(),
-            urn.clone(),
-        )
-        .await?
-        {
+        let at = match at {
             Some(at) => at,
             None => return Ok(()),
         };
-- 
2.32.0 (Apple Git-132)

Re: [PATCH radicle-link v4 1/3] Add integration test for bins and fix git version checks

Alex Good <alex@memoryandthought.me>

Details

Message ID: <CAH_DpYQP7Vc2FLJee8FH_6rgTkz246VV1n-LZQUzck1i=owiKA@mail.gmail.com>
In-Reply-To: <20220825054156.57232-2-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On 24/08/22 10:41pm, Han Xu wrote:
> diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
> +enum LnkCmd {
> +    ProfileCreate,
> +    ProfileGet,
> +    ProfilePeer,
> +    ProfileSshAdd,
> +    IdPersonCreate(String),  // the associated string is "the person's name".
> +    IdLocalSet(String),      // the associated string is "urn".
> +    IdProjectCreate(String), // the associated string is "the project name".
> +    Clone(String, String),   // the associated string is "urn", "peer_id"
> +}
> +
> +/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
> +/// Return.0: true if this is the parent (i.e. test) process,
> +///           false if this is the child (i.e. lnk) process.
> +/// Return.1: an output that depends on the `cmd`.
> +fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {

I still find this return and switch on the returned `bool` a bit
cumbersome. Is it worth using something like
https://docs.rs/pty-process/latest/pty_process/struct.Child.html
which would give us basically the same API as `std::process::Child`?

Failing that we could write a little macro to remove all the `is_parent`
checks in the calling code.

> +fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
> +    let port = "9987";
> +    let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
> +    let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
> +    let target_dir = bins_target_dir();
> +    let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
> +
> +    Command::new("cargo")
> +        .arg("build")
> +        .arg("--target-dir")
> +        .arg(&target_dir)
> +        .arg("--manifest-path")
> +        .arg(manifest_path)
> +        .arg("-p")
> +        .arg("lnk-gitd")
> +        .stdout(Stdio::inherit())
> +        .output()
> +        .expect("cargo build lnk-gitd failed");
> +
> +        Command::new("systemd-socket-activate")
> +            .arg("-l")
> +            .arg(port)
> +            .arg("--fdname=ssh")
> +            .arg("-E")
> +            .arg("SSH_AUTH_SOCK")
> +            .arg("-E")
> +            .arg("RUST_BACKTRACE")
> +            .arg(&exec_path)
> +            .arg(lnk_home)
> +            .arg("--linkd-rpc-socket")
> +            .arg(rpc_socket)
> +            .arg("--push-seeds")
> +            .arg("--fetch-seeds")
> +            .arg("--linger-timeout")
> +            .arg("10000")
> +            .spawn()
> +            .expect("lnk-gitd failed to start");

This is platform dependent (i.e. OSX doesn't have
`systemd-socket-activate`). We also don't really need to use the socket
activated version of this, we could just run `lnk_home --push-seeds
--fetch-seeds --addr 127.0.0.1:<some port>` (i.e. no `--linger-timeout`)
which will immediately run `lnk-gitd` and keep it running until it's
killed.

Re: [PATCH radicle-link v4 1/3] Add integration test for bins and fix git version checks

<MissM_signed@protonmail.ch>

Details

Message ID: <YJO9BUK_HUlkBhSsrb4FQfkM3yfC8Luxcl5Au7xnrx7Em5N5-zRKq9zwieQCZukLbS6w7lAwaBQ1Xhu9x4JIq20oVJDPJgQcvVErW8SIdNo=@protonmail.ch>
In-Reply-To: <CAH_DpYQP7Vc2FLJee8FH_6rgTkz246VV1n-LZQUzck1i=owiKA@mail.gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

So I am trying to get some collaboration on testing going between testing CLI type binaries across

Is there a reason none of these could be used ?

https://crates.io/crates/trycmd
https://crates.io/crates/term-transcript
out of date: https://github.com/rust-rspec/rspec
https://crates.io/crates/trybuild
https://nexte.st/book/leaky-tests.html

compiletest seems to have detached yay https://github.com/Manishearth/compiletest-rs
and ofc insta https://crates.io/crates/insta
https://crates.io/crates/snapbox
https://crates.io/crates/assert_cmd, https://crates.io/crates/assert_fs
https://crates.io/crates/duct
https://crates.io/crates/rexpect
https://crates.io/crates/dir-diff

I use insta a lot myself

Rust idiomatic way is to use small crates from the ecosystem

If we are still going ahead of using Rust to do integration tests.

------- Original Message -------
On Friday, August 26th, 2022 at 1:44 AM, Alex Good <alex@memoryandthought.me> wrote:


> 

> 

> On 24/08/22 10:41pm, Han Xu wrote:
> 

> > diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
> > +enum LnkCmd {
> > + ProfileCreate,
> > + ProfileGet,
> > + ProfilePeer,
> > + ProfileSshAdd,
> > + IdPersonCreate(String), // the associated string is "the person's name".
> > + IdLocalSet(String), // the associated string is "urn".
> > + IdProjectCreate(String), // the associated string is "the project name".
> > + Clone(String, String), // the associated string is "urn", "peer_id"
> > +}
> > +
> > +/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
> > +/// Return.0: true if this is the parent (i.e. test) process,
> > +/// false if this is the child (i.e. lnk) process.
> > +/// Return.1: an output that depends on the `cmd`.
> > +fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
> 

> 

> I still find this return and switch on the returned `bool` a bit
> cumbersome. Is it worth using something like
> https://docs.rs/pty-process/latest/pty_process/struct.Child.html
> which would give us basically the same API as `std::process::Child`?
> 

> Failing that we could write a little macro to remove all the `is_parent`
> checks in the calling code.
> 

> > +fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
> > + let port = "9987";
> > + let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
> > + let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
> > + let target_dir = bins_target_dir();
> > + let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
> > +
> > + Command::new("cargo")
> > + .arg("build")
> > + .arg("--target-dir")
> > + .arg(&target_dir)
> > + .arg("--manifest-path")
> > + .arg(manifest_path)
> > + .arg("-p")
> > + .arg("lnk-gitd")
> > + .stdout(Stdio::inherit())
> > + .output()
> > + .expect("cargo build lnk-gitd failed");
> > +
> > + Command::new("systemd-socket-activate")
> > + .arg("-l")
> > + .arg(port)
> > + .arg("--fdname=ssh")
> > + .arg("-E")
> > + .arg("SSH_AUTH_SOCK")
> > + .arg("-E")
> > + .arg("RUST_BACKTRACE")
> > + .arg(&exec_path)
> > + .arg(lnk_home)
> > + .arg("--linkd-rpc-socket")
> > + .arg(rpc_socket)
> > + .arg("--push-seeds")
> > + .arg("--fetch-seeds")
> > + .arg("--linger-timeout")
> > + .arg("10000")
> > + .spawn()
> > + .expect("lnk-gitd failed to start");
> 

> 

> This is platform dependent (i.e. OSX doesn't have
> `systemd-socket-activate`). We also don't really need to use the socket
> activated version of this, we could just run `lnk_home --push-seeds --fetch-seeds --addr 127.0.0.1:<some port>` (i.e. no `--linger-timeout`)
> 

> which will immediately run `lnk-gitd` and keep it running until it's
> killed.

Re: [PATCH radicle-link v4 1/3] Add integration test for bins and fix git version checks

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CMF9O74XH11C.3C0YFPTXOFFUD@haptop>
In-Reply-To: <YJO9BUK_HUlkBhSsrb4FQfkM3yfC8Luxcl5Au7xnrx7Em5N5-zRKq9zwieQCZukLbS6w7lAwaBQ1Xhu9x4JIq20oVJDPJgQcvVErW8SIdNo=@protonmail.ch> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Thu Aug 25, 2022 at 5:10 PM IST,  wrote:
> So I am trying to get some collaboration on testing going between testing CLI type binaries across
>
> Is there a reason none of these could be used ?
>
> https://crates.io/crates/trycmd
> https://crates.io/crates/term-transcript
> out of date: https://github.com/rust-rspec/rspec
> https://crates.io/crates/trybuild
> https://nexte.st/book/leaky-tests.html
>
> compiletest seems to have detached yay https://github.com/Manishearth/compiletest-rs
> and ofc insta https://crates.io/crates/insta
> https://crates.io/crates/snapbox
> https://crates.io/crates/assert_cmd, https://crates.io/crates/assert_fs
> https://crates.io/crates/duct
> https://crates.io/crates/rexpect
> https://crates.io/crates/dir-diff
>
> I use insta a lot myself
>
> Rust idiomatic way is to use small crates from the ecosystem
>
> If we are still going ahead of using Rust to do integration tests.

Which ones do you find useful/have tried?

How would they be used to change over the code that Han is providing?

Re: [PATCH radicle-link v4 1/3] Add integration test for bins and fix git version checks

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqfNP7yr3R-wTHUFTni-p5apkmFwFvGthneMoeJ7qNVK+Q@mail.gmail.com>
In-Reply-To: <CAH_DpYQP7Vc2FLJee8FH_6rgTkz246VV1n-LZQUzck1i=owiKA@mail.gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Thu, Aug 25, 2022 at 8:44 AM Alex Good <alex@memoryandthought.me> wrote:
>
> On 24/08/22 10:41pm, Han Xu wrote:
> > diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
> > +enum LnkCmd {
> > +    ProfileCreate,
> > +    ProfileGet,
> > +    ProfilePeer,
> > +    ProfileSshAdd,
> > +    IdPersonCreate(String),  // the associated string is "the person's name".
> > +    IdLocalSet(String),      // the associated string is "urn".
> > +    IdProjectCreate(String), // the associated string is "the project name".
> > +    Clone(String, String),   // the associated string is "urn", "peer_id"
> > +}
> > +
> > +/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
> > +/// Return.0: true if this is the parent (i.e. test) process,
> > +///           false if this is the child (i.e. lnk) process.
> > +/// Return.1: an output that depends on the `cmd`.
> > +fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
>
> I still find this return and switch on the returned `bool` a bit
> cumbersome. Is it worth using something like
> https://docs.rs/pty-process/latest/pty_process/struct.Child.html
> which would give us basically the same API as `std::process::Child`?
>
> Failing that we could write a little macro to remove all the `is_parent`
> checks in the calling code.

Thanks for your suggestion. I've created a macro to replace the run_lnk fn
and removed all `is_parent` checks.

>
> > +fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
> > +    let port = "9987";
> > +    let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
> > +    let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
> > +    let target_dir = bins_target_dir();
> > +    let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
> > +
> > +    Command::new("cargo")
> > +        .arg("build")
> > +        .arg("--target-dir")
> > +        .arg(&target_dir)
> > +        .arg("--manifest-path")
> > +        .arg(manifest_path)
> > +        .arg("-p")
> > +        .arg("lnk-gitd")
> > +        .stdout(Stdio::inherit())
> > +        .output()
> > +        .expect("cargo build lnk-gitd failed");
> > +
> > +        Command::new("systemd-socket-activate")
> > +            .arg("-l")
> > +            .arg(port)
> > +            .arg("--fdname=ssh")
> > +            .arg("-E")
> > +            .arg("SSH_AUTH_SOCK")
> > +            .arg("-E")
> > +            .arg("RUST_BACKTRACE")
> > +            .arg(&exec_path)
> > +            .arg(lnk_home)
> > +            .arg("--linkd-rpc-socket")
> > +            .arg(rpc_socket)
> > +            .arg("--push-seeds")
> > +            .arg("--fetch-seeds")
> > +            .arg("--linger-timeout")
> > +            .arg("10000")
> > +            .spawn()
> > +            .expect("lnk-gitd failed to start");
>
> This is platform dependent (i.e. OSX doesn't have
> `systemd-socket-activate`). We also don't really need to use the socket
> activated version of this, we could just run `lnk_home --push-seeds
> --fetch-seeds --addr 127.0.0.1:<some port>` (i.e. no `--linger-timeout`)
> which will immediately run `lnk-gitd` and keep it running until it's
> killed.

Good to know that. I've updated the code to remove the use of
systemd-socket-activate.
Will send out the new version of the patch.

Thanks!
Han

[PATCH radicle-link v5 0/5] Add integration test for bins and fix bugs found

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220825210251.61675-1-keepsimple@gmail.com>
In-Reply-To: <20220825054156.57232-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

This is the 5th version (re-rolling) patch to add an integration test for `bins` package.

It is built on top of version 4 patch and addressed review comments (see new commits):
 - created a new macro to simplify the test code.
 - removed the use of systemd-socket-activate.

The test case scenario follows the steps in https://github.com/alexjg/linkd-playground ,
except it is in Rust now and we can just run `cargo test`. To see a more helpful output,
please run `cargo test -- --nocapture` from the `bins/tests` directory.

We also found two bugs along the way: 

1. Git version checks are off. In our testing, it shows that
git 2.25.1 on Ubuntu 20.04 and git 2.30.2 on Debian 11 do not need the workaround that
adds "namespaces" manually. Hence fixing the git version checked against.

2. `request_pull` to the seed fails to pull the latest commit. The reason being that
the signed_refs for `heads/master` is not the latest. A fix is provided.

Han Xu (5):
  Add integration test for bins and fix git version checks
  check peer1 and peer2 ref/heads/master
  fix post_receive to send updated refs in requst_pull
  Use macro to refactor run_lnk
  remove the use of systemd-socket-activate

 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 417 +++++++++++++++++++++++++++++++++
 cli/gitd-lib/src/hooks.rs      |  19 +-
 link-git/src/protocol/fetch.rs |   5 +-
 link-git/src/protocol/ls.rs    |   5 +-
 7 files changed, 479 insertions(+), 10 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v5 1/5] Add integration test for bins and fix git version checks

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220825210251.61675-2-keepsimple@gmail.com>
In-Reply-To: <20220825210251.61675-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +500 -2

---
 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 449 +++++++++++++++++++++++++++++++++
 link-git/src/protocol/fetch.rs |   5 +-
 link-git/src/protocol/ls.rs    |   5 +-
 6 files changed, 500 insertions(+), 2 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

diff --git a/bins/Cargo.lock b/bins/Cargo.lock
index d5756144..4d0ea413 100644
--- a/bins/Cargo.lock
@@ -838,6 +838,17 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "errno"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e2b2decb0484e15560df3210cf0d78654bb0864b2c138977c07e377a1bae0e2"
+dependencies = [
+ "kernel32-sys",
+ "libc",
+ "winapi 0.2.8",
+]
+
 [[package]]
 name = "event-listener"
 version = "2.5.2"
@@ -2945,6 +2956,16 @@ dependencies = [
  "human_format",
 ]
 
+[[package]]
+name = "pty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50f3d255966981eb4e4c5df3e983e6f7d163221f547406d83b6a460ff5c5ee8"
+dependencies = [
+ "errno",
+ "libc",
+]
+
 [[package]]
 name = "quanta"
 version = "0.4.1"
@@ -3567,6 +3588,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "tests"
+version = "0.1.0"
+dependencies = [
+ "pty",
+ "serde_json",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
diff --git a/bins/Cargo.toml b/bins/Cargo.toml
index 3d1786f4..99b3c69f 100644
--- a/bins/Cargo.toml
@@ -5,4 +5,5 @@ members = [
   "lnk-gitd",
   "lnk-identities-dev",
   "lnk-profile-dev",
+  "tests",
 ]
diff --git a/bins/tests/Cargo.toml b/bins/tests/Cargo.toml
new file mode 100644
index 00000000..3ae82769
--- /dev/null
@@ -0,0 +1,13 @@
+[package]
+name = "tests"
+version = "0.1.0"
+authors = ["Han Xu <keepsimple@gmail.com>"]
+edition = "2021"
+
+[dev-dependencies]
+pty = "0.2"
+serde_json = "1.0"
+
+[[test]]
+name = "integration_test"
+path = "integration_test.rs"
diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
new file mode 100644
index 00000000..770f1b5e
--- /dev/null
@@ -0,0 +1,449 @@
+// Copyright © 2022 The Radicle Link Contributors
+//
+// This file is part of radicle-link, distributed under the GPLv3 with Radicle
+// Linking Exception. For full terms see the included LICENSE file.
+
+//! Integration test to exercise the programs in `bins`.
+
+use pty::fork::*;
+use serde_json::{json, Value};
+use std::{
+    env,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    process::{Child, Command, Stdio},
+    thread,
+    time::{Duration, SystemTime},
+};
+
+/// This test is inspired by https://github.com/alexjg/linkd-playground
+///
+/// Tests a typical scenario: there are two peer nodes and one seed node.
+/// The main steps are:
+///   - Setup a profile for each node in tmp home directories.
+///   - Setup SSH keys for each profile.
+///   - Create identities.
+///   - Create a local repo for peer 1.
+///   - Start `linkd` for the seed, and `lnk-gitd` for peer 1.
+///   - Push peer 1 repo to its monorepo and to the seed.
+///   - Clone the peer 1 repo to peer 2 via seed.
+#[test]
+fn two_peers_and_a_seed() {
+    let peer1_home = "/tmp/link-local-1";
+    let peer2_home = "/tmp/link-local-2";
+    let seed_home = "/tmp/seed-home";
+    let passphrase = b"play\n";
+
+    println!("\n== create lnk homes for two peers and one seed ==\n");
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Creating local link 1 identity ==\n");
+    let peer1_name = "sockpuppet1".to_string();
+    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer1_name), peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let urn1 = v["urn"].as_str().unwrap().to_string();
+    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn1), peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Creating local link 2 identity ==\n");
+    let peer2_name = "sockpuppet2".to_string();
+    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer2_name), peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let urn2 = v["urn"].as_str().unwrap().to_string();
+    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn2), peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Create a local repository ==\n");
+    let peer1_proj = format!("peer1_proj_{}", timestamp());
+    let (is_parent, output) = run_lnk(
+        LnkCmd::IdProjectCreate(peer1_proj.clone()),
+        peer1_home,
+        passphrase,
+    );
+    if !is_parent {
+        return;
+    }
+    let v: Value = serde_json::from_str(&output).unwrap();
+    let proj_urn = v["urn"].as_str().unwrap().to_string();
+    println!("our project URN: {}", &proj_urn);
+
+    println!("\n== Add the seed to the local peer seed configs ==\n");
+    let (is_parent, seed_peer_id) = run_lnk(LnkCmd::ProfilePeer, seed_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let seed_endpoint = format!("{}@127.0.0.1:8799", &seed_peer_id);
+
+    let (is_parent, peer1_profile) = run_lnk(LnkCmd::ProfileGet, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
+    let mut peer1_f = File::create(peer1_seed).unwrap();
+    peer1_f.write_all(seed_endpoint.as_bytes()).unwrap();
+
+    let (is_parent, peer2_profile) = run_lnk(LnkCmd::ProfileGet, peer2_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
+    let mut peer2_f = File::create(peer2_seed).unwrap();
+    peer2_f.write_all(seed_endpoint.as_bytes()).unwrap();
+
+    println!("\n== Start the seed linkd ==\n");
+    let manifest_path = manifest_path();
+    let mut linkd = spawn_linkd(seed_home, &manifest_path);
+
+    println!("\n== Start the peer 1 gitd ==\n");
+    let (is_parent, peer1_peer_id) = run_lnk(LnkCmd::ProfilePeer, peer1_home, passphrase);
+    if !is_parent {
+        return;
+    }
+    spawn_lnk_gitd(peer1_home, &manifest_path, &peer1_peer_id);
+
+    println!("\n== Make some changes in the repo ==\n");
+    env::set_current_dir(&peer1_proj).unwrap();
+    let mut test_file = File::create("test").unwrap();
+    test_file.write_all(b"test").unwrap();
+    Command::new("git")
+        .arg("add")
+        .arg("test")
+        .output()
+        .expect("failed to do git add");
+    let output = Command::new("git")
+        .arg("commit")
+        .arg("-m")
+        .arg("test commit")
+        .output()
+        .expect("failed to do git commit");
+    println!("git-commit: {:?}", &output);
+
+    println!("\n== Add the linkd remote to the repo ==\n");
+    let remote_url = format!("ssh://rad@127.0.0.1:9987/{}.git", &proj_urn);
+    Command::new("git")
+        .arg("remote")
+        .arg("add")
+        .arg("linkd")
+        .arg(remote_url)
+        .output()
+        .expect("failed to do git remote add");
+
+    clean_up_known_hosts();
+
+    let is_parent = run_git_push();
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Clone to peer2 ==\n");
+
+    env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
+    let (is_parent, _) = run_lnk(
+        LnkCmd::Clone(proj_urn, peer1_peer_id),
+        peer2_home,
+        passphrase,
+    );
+    if !is_parent {
+        return;
+    }
+
+    println!("\n== Kill linkd (seed) ==\n");
+
+    linkd.kill().ok();
+}
+
+enum LnkCmd {
+    ProfileCreate,
+    ProfileGet,
+    ProfilePeer,
+    ProfileSshAdd,
+    IdPersonCreate(String),  // the associated string is "the person's name".
+    IdLocalSet(String),      // the associated string is "urn".
+    IdProjectCreate(String), // the associated string is "the project name".
+    Clone(String, String),   // the associated string is "urn", "peer_id"
+}
+
+/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
+/// Return.0: true if this is the parent (i.e. test) process,
+///           false if this is the child (i.e. lnk) process.
+/// Return.1: an output that depends on the `cmd`.
+fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        // Input the passphrase if necessary.
+        match cmd {
+            LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
+                parent.write_all(passphrase).unwrap();
+                println!("{}: wrote passphase", lnk_home);
+            },
+            _ => {},
+        }
+
+        // Print the output and decode them if necessary.
+        let buf_reader = BufReader::new(parent);
+        let mut output = String::new();
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("{}: {}", lnk_home, line);
+
+            match cmd {
+                LnkCmd::IdPersonCreate(ref _name) => {
+                    if line.find("\"urn\":").is_some() {
+                        output = line; // get the line with URN.
+                    }
+                },
+                LnkCmd::IdProjectCreate(ref _name) => {
+                    if line.find("\"urn\":").is_some() {
+                        output = line; // get the line with URN.
+                    }
+                },
+                LnkCmd::ProfileGet => {
+                    output = line; // get the last line for profile id.
+                },
+                LnkCmd::ProfilePeer => {
+                    output = line; // get the last line for peer id.
+                },
+                LnkCmd::Clone(ref _urn, ref _peer) => {
+                    output = line;
+                },
+                _ => {},
+            }
+        }
+
+        (true, output)
+    } else {
+        // Child process is to run `lnk`.
+        let manifest_path = manifest_path();
+
+        // cargo run \
+        // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
+        // -p lnk -- "$@"
+        let mut lnk_cmd = Command::new("cargo");
+        lnk_cmd
+            .env("LNK_HOME", lnk_home)
+            .arg("run")
+            .arg("--manifest-path")
+            .arg(manifest_path)
+            .arg("-p")
+            .arg("lnk")
+            .arg("--");
+        let full_cmd = match cmd {
+            LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
+            LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
+            LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
+            LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
+            LnkCmd::IdPersonCreate(name) => {
+                let payload = json!({ "name": name });
+                lnk_cmd
+                    .arg("identities")
+                    .arg("person")
+                    .arg("create")
+                    .arg("new")
+                    .arg("--payload")
+                    .arg(payload.to_string())
+            },
+            LnkCmd::IdLocalSet(urn) => lnk_cmd
+                .arg("identities")
+                .arg("local")
+                .arg("set")
+                .arg("--urn")
+                .arg(urn),
+            LnkCmd::IdProjectCreate(name) => {
+                let payload = json!({"name": name, "default_branch": "master"});
+                let project_path = format!("./{}", name);
+                lnk_cmd
+                    .arg("identities")
+                    .arg("project")
+                    .arg("create")
+                    .arg("new")
+                    .arg("--path")
+                    .arg(project_path)
+                    .arg("--payload")
+                    .arg(payload.to_string())
+            },
+            LnkCmd::Clone(urn, peer_id) => {
+                let peer2_proj = format!("peer2_proj_{}", timestamp());
+                lnk_cmd
+                    .arg("clone")
+                    .arg("--urn")
+                    .arg(urn)
+                    .arg("--path")
+                    .arg(peer2_proj)
+                    .arg("--peer")
+                    .arg(peer_id)
+            },
+        };
+        full_cmd.status().expect("lnk cmd failed:");
+
+        (false, String::new())
+    }
+}
+
+fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
+    let log_name = format!("linkd_{}.log", &timestamp());
+    let log_file = File::create(&log_name).unwrap();
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/linkd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("linkd")
+        .output()
+        .expect("cargo build linkd failed");
+
+    let child = Command::new(&exec_path)
+        .env("RUST_BACKTRACE", "1")
+        .arg("--lnk-home")
+        .arg(lnk_home)
+        .arg("--track")
+        .arg("everything")
+        .arg("--protocol-listen")
+        .arg("127.0.0.1:8799")
+        .stdout(Stdio::from(log_file))
+        .spawn()
+        .expect("linkd failed to start");
+    println!("linkd stdout redirected to {}", &log_name);
+    thread::sleep(Duration::from_secs(1));
+    child
+}
+
+fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
+    let port = "9987";
+    let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
+    let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk-gitd")
+        .stdout(Stdio::inherit())
+        .output()
+        .expect("cargo build lnk-gitd failed");
+
+        Command::new("systemd-socket-activate")
+            .arg("-l")
+            .arg(port)
+            .arg("--fdname=ssh")
+            .arg("-E")
+            .arg("SSH_AUTH_SOCK")
+            .arg("-E")
+            .arg("RUST_BACKTRACE")
+            .arg(&exec_path)
+            .arg(lnk_home)
+            .arg("--linkd-rpc-socket")
+            .arg(rpc_socket)
+            .arg("--push-seeds")
+            .arg("--fetch-seeds")
+            .arg("--linger-timeout")
+            .arg("10000")
+            .spawn()
+            .expect("lnk-gitd failed to start");
+            println!("started lnk-gitd");
+
+}
+
+/// Returns true if this is the parent process,
+/// returns false if this is the child process.
+fn run_git_push() -> bool {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        let yes = b"yes\n";
+        let buf_reader = BufReader::new(parent);
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("git-push: {}", line);
+            if line.find("key fingerprint").is_some() {
+                parent.write_all(yes).unwrap();
+            }
+        }
+
+        true
+    } else {
+        Command::new("git")
+            .arg("push")
+            .arg("linkd")
+            .status()
+            .expect("failed to do git push");
+        false
+    }
+}
+
+/// Returns UNIX_TIME in millis.
+fn timestamp() -> u128 {
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap();
+    now.as_millis()
+}
+
+/// Returns the full path of `bins` manifest file.
+fn manifest_path() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/Cargo.toml", package_dir.strip_suffix("/tests").unwrap())
+}
+
+/// Returns the full path of `bins/target`.
+fn bins_target_dir() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/target", package_dir.strip_suffix("/tests").unwrap())
+}
+
+fn clean_up_known_hosts() {
+    // ssh-keygen -f "/home/pi/.ssh/known_hosts" -R "[127.0.0.1]:9987"
+    let home_dir = env!("HOME");
+    let known_hosts = format!("{}/.ssh/known_hosts", &home_dir);
+    let output = Command::new("ssh-keygen")
+        .arg("-f")
+        .arg(known_hosts)
+        .arg("-R")
+        .arg("[127.0.0.1]:9987")
+        .output()
+        .expect("failed to do ssh-keygen");
+    println!("ssh-keygen: {:?}", &output);
+}
diff --git a/link-git/src/protocol/fetch.rs b/link-git/src/protocol/fetch.rs
index a5a73065..98b92b51 100644
--- a/link-git/src/protocol/fetch.rs
+++ b/link-git/src/protocol/fetch.rs
@@ -38,8 +38,11 @@ use super::{packwriter::PackWriter, remote_git_version, transport};
 //
 // cf. https://lore.kernel.org/git/CD2XNXHACAXS.13J6JTWZPO1JA@schmidt/
 // Fixed in `git.git` 1ab13eb, which should land in 2.34
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace_want_ref(caps: &client::Capabilities) -> bool {
-    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.33.0").unwrap());
+    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version <= *FIXED_AFTER)
diff --git a/link-git/src/protocol/ls.rs b/link-git/src/protocol/ls.rs
index b3516454..de95b4a3 100644
--- a/link-git/src/protocol/ls.rs
+++ b/link-git/src/protocol/ls.rs
@@ -22,9 +22,12 @@ use super::{remote_git_version, transport};
 // Work around `git-upload-pack` not handling namespaces properly
 //
 // cf. https://lore.kernel.org/git/pMV5dJabxOBTD8kJBaPuWK0aS6OJhRQ7YFGwfhPCeSJEbPDrIFBza36nXBCgUCeUJWGmpjPI1rlOGvZJEh71Ruz4SqljndUwOCoBUDRHRDU=@eagain.st/
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace(caps: &client::Capabilities) -> bool {
     static MIN_GIT_VERSION_NAMESPACES: Lazy<Version> =
-        Lazy::new(|| Version::new("2.31.0").unwrap());
+        Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version < *MIN_GIT_VERSION_NAMESPACES)
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v5 2/5] check peer1 and peer2 ref/heads/master

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220825210251.61675-3-keepsimple@gmail.com>
In-Reply-To: <20220825210251.61675-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +25 -6

---
 bins/tests/integration_test.rs | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
index 770f1b5e..ea6815ab 100644
--- a/bins/tests/integration_test.rs
@@ -170,11 +170,15 @@ fn two_peers_and_a_seed() {
         return;
     }
 
+    let peer1_last_commit = git_last_commit();
+    println!("\n== peer1 project last commit: {} ==\n", &peer1_last_commit);
+
     println!("\n== Clone to peer2 ==\n");
 
     env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
+    let peer2_proj = format!("peer2_proj_{}", timestamp());
     let (is_parent, _) = run_lnk(
-        LnkCmd::Clone(proj_urn, peer1_peer_id),
+        LnkCmd::Clone(proj_urn, peer1_peer_id, peer2_proj.clone()),
         peer2_home,
         passphrase,
     );
@@ -182,9 +186,16 @@ fn two_peers_and_a_seed() {
         return;
     }
 
-    println!("\n== Kill linkd (seed) ==\n");
+    env::set_current_dir(peer2_proj).unwrap();
+    let peer2_last_commit = git_last_commit();
+    println!("\n== peer1 proj last commit: {}", &peer1_last_commit);
+    println!("\n== peer2 proj last commit: {}", &peer2_last_commit);
+
+    println!("\n== Cleanup: kill linkd (seed) ==\n");
 
     linkd.kill().ok();
+
+    assert_eq!(peer1_last_commit, peer2_last_commit);
 }
 
 enum LnkCmd {
@@ -195,7 +206,7 @@ enum LnkCmd {
     IdPersonCreate(String),  // the associated string is "the person's name".
     IdLocalSet(String),      // the associated string is "urn".
     IdProjectCreate(String), // the associated string is "the project name".
-    Clone(String, String),   // the associated string is "urn", "peer_id"
+    Clone(String, String, String),   // the associated string is "urn", "peer_id", "path"
 }
 
 /// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
@@ -238,7 +249,7 @@ fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
                 LnkCmd::ProfilePeer => {
                     output = line; // get the last line for peer id.
                 },
-                LnkCmd::Clone(ref _urn, ref _peer) => {
+                LnkCmd::Clone(ref _urn, ref _peer, ref _path) => {
                     output = line;
                 },
                 _ => {},
@@ -296,8 +307,7 @@ fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
                     .arg("--payload")
                     .arg(payload.to_string())
             },
-            LnkCmd::Clone(urn, peer_id) => {
-                let peer2_proj = format!("peer2_proj_{}", timestamp());
+            LnkCmd::Clone(urn, peer_id, peer2_proj) => {
                 lnk_cmd
                     .arg("clone")
                     .arg("--urn")
@@ -414,6 +424,15 @@ fn run_git_push() -> bool {
     }
 }
 
+fn git_last_commit() -> String {
+    let output = Command::new("git")
+        .arg("rev-parse")
+        .arg("HEAD")
+        .output()
+        .expect("failed to run git rev-parse");
+    String::from_utf8_lossy(&output.stdout).to_string()
+}
+
 /// Returns UNIX_TIME in millis.
 fn timestamp() -> u128 {
     let now = SystemTime::now()
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v5 3/5] fix post_receive to send updated refs in requst_pull

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220825210251.61675-4-keepsimple@gmail.com>
In-Reply-To: <20220825210251.61675-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +11 -8

---
 cli/gitd-lib/src/hooks.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/cli/gitd-lib/src/hooks.rs b/cli/gitd-lib/src/hooks.rs
index 5e928ffd..cf37d71f 100644
--- a/cli/gitd-lib/src/hooks.rs
+++ b/cli/gitd-lib/src/hooks.rs
@@ -77,6 +77,16 @@ where
         E: std::error::Error + Send + 'static,
         P: ProgressReporter<Error = E>,
     {
+        // update the signed refs before a possible request_pull,
+        // so that the peer can receive the latest refs.
+        let at = update_signed_refs(
+            reporter,
+            self.spawner.clone(),
+            self.pool.clone(),
+            urn.clone(),
+        )
+        .await?;
+
         if self.post_receive.request_pull {
             tracing::info!("executing request-pull");
             request_pull(reporter, &self.client, &self.seeds, urn.clone()).await?;
@@ -87,14 +97,7 @@ where
             )
             .await?;
         }
-        let at = match update_signed_refs(
-            reporter,
-            self.spawner.clone(),
-            self.pool.clone(),
-            urn.clone(),
-        )
-        .await?
-        {
+        let at = match at {
             Some(at) => at,
             None => return Ok(()),
         };
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v5 4/5] Use macro to refactor run_lnk

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220825210251.61675-5-keepsimple@gmail.com>
In-Reply-To: <20220825210251.61675-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +158 -195

---
 bins/tests/integration_test.rs | 353 +++++++++++++++------------------
 1 file changed, 158 insertions(+), 195 deletions(-)

diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
index ea6815ab..43de968a 100644
--- a/bins/tests/integration_test.rs
@@ -35,92 +35,55 @@ fn two_peers_and_a_seed() {
     let passphrase = b"play\n";
 
     println!("\n== create lnk homes for two peers and one seed ==\n");
-    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer1_home, passphrase);
-    if !is_parent {
-        return;
-    }
-    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, peer2_home, passphrase);
-    if !is_parent {
-        return;
-    }
-    let (is_parent, _) = run_lnk(LnkCmd::ProfileCreate, seed_home, passphrase);
-    if !is_parent {
-        return;
-    }
+    let cmd = LnkCmd::ProfileCreate;
+    run_lnk!(&cmd, peer1_home, passphrase);
+    run_lnk!(&cmd, peer2_home, passphrase);
+    run_lnk!(&cmd, seed_home, passphrase);
 
     println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
-    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer1_home, passphrase);
-    if !is_parent {
-        return;
-    }
-    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, peer2_home, passphrase);
-    if !is_parent {
-        return;
-    }
-    let (is_parent, _) = run_lnk(LnkCmd::ProfileSshAdd, seed_home, passphrase);
-    if !is_parent {
-        return;
-    }
+    let cmd = LnkCmd::ProfileSshAdd;
+    run_lnk!(&cmd, peer1_home, passphrase);
+    run_lnk!(&cmd, peer2_home, passphrase);
+    run_lnk!(&cmd, seed_home, passphrase);
 
     println!("\n== Creating local link 1 identity ==\n");
     let peer1_name = "sockpuppet1".to_string();
-    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer1_name), peer1_home, passphrase);
-    if !is_parent {
-        return;
-    }
+    let cmd = LnkCmd::IdPersonCreate(peer1_name);
+    let output = run_lnk!(&cmd, peer1_home, passphrase);
     let v: Value = serde_json::from_str(&output).unwrap();
     let urn1 = v["urn"].as_str().unwrap().to_string();
-    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn1), peer1_home, passphrase);
-    if !is_parent {
-        return;
-    }
+    let cmd = LnkCmd::IdLocalSet(urn1);
+    run_lnk!(&cmd, peer1_home, passphrase);
 
     println!("\n== Creating local link 2 identity ==\n");
     let peer2_name = "sockpuppet2".to_string();
-    let (is_parent, output) = run_lnk(LnkCmd::IdPersonCreate(peer2_name), peer2_home, passphrase);
-    if !is_parent {
-        return;
-    }
+    let cmd = LnkCmd::IdPersonCreate(peer2_name);
+    let output = run_lnk!(&cmd, peer2_home, passphrase);
     let v: Value = serde_json::from_str(&output).unwrap();
     let urn2 = v["urn"].as_str().unwrap().to_string();
-    let (is_parent, _) = run_lnk(LnkCmd::IdLocalSet(urn2), peer2_home, passphrase);
-    if !is_parent {
-        return;
-    }
+    let cmd = LnkCmd::IdLocalSet(urn2);
+    run_lnk!(&cmd, peer2_home, passphrase);
 
     println!("\n== Create a local repository ==\n");
     let peer1_proj = format!("peer1_proj_{}", timestamp());
-    let (is_parent, output) = run_lnk(
-        LnkCmd::IdProjectCreate(peer1_proj.clone()),
-        peer1_home,
-        passphrase,
-    );
-    if !is_parent {
-        return;
-    }
+    let cmd = LnkCmd::IdProjectCreate(peer1_proj.clone());
+    let output = run_lnk!(&cmd, peer1_home, passphrase);
     let v: Value = serde_json::from_str(&output).unwrap();
     let proj_urn = v["urn"].as_str().unwrap().to_string();
     println!("our project URN: {}", &proj_urn);
 
     println!("\n== Add the seed to the local peer seed configs ==\n");
-    let (is_parent, seed_peer_id) = run_lnk(LnkCmd::ProfilePeer, seed_home, passphrase);
-    if !is_parent {
-        return;
-    }
+    let cmd = LnkCmd::ProfilePeer;
+    let seed_peer_id = run_lnk!(&cmd, seed_home, passphrase);
     let seed_endpoint = format!("{}@127.0.0.1:8799", &seed_peer_id);
 
-    let (is_parent, peer1_profile) = run_lnk(LnkCmd::ProfileGet, peer1_home, passphrase);
-    if !is_parent {
-        return;
-    }
+    let cmd = LnkCmd::ProfileGet;
+    let peer1_profile = run_lnk!(&cmd, peer1_home, passphrase);
     let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
     let mut peer1_f = File::create(peer1_seed).unwrap();
     peer1_f.write_all(seed_endpoint.as_bytes()).unwrap();
 
-    let (is_parent, peer2_profile) = run_lnk(LnkCmd::ProfileGet, peer2_home, passphrase);
-    if !is_parent {
-        return;
-    }
+    let peer2_profile = run_lnk!(&cmd, peer2_home, passphrase);
     let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
     let mut peer2_f = File::create(peer2_seed).unwrap();
     peer2_f.write_all(seed_endpoint.as_bytes()).unwrap();
@@ -130,10 +93,8 @@ fn two_peers_and_a_seed() {
     let mut linkd = spawn_linkd(seed_home, &manifest_path);
 
     println!("\n== Start the peer 1 gitd ==\n");
-    let (is_parent, peer1_peer_id) = run_lnk(LnkCmd::ProfilePeer, peer1_home, passphrase);
-    if !is_parent {
-        return;
-    }
+    let cmd = LnkCmd::ProfilePeer;
+    let peer1_peer_id = run_lnk!(&cmd, peer1_home, passphrase);
     spawn_lnk_gitd(peer1_home, &manifest_path, &peer1_peer_id);
 
     println!("\n== Make some changes in the repo ==\n");
@@ -171,17 +132,17 @@ fn two_peers_and_a_seed() {
     }
 
     let peer1_last_commit = git_last_commit();
-    println!("\n== peer1 project last commit: {} ==\n", &peer1_last_commit);
+    println!(
+        "\n== peer1 project last commit: {} ==\n",
+        &peer1_last_commit
+    );
 
     println!("\n== Clone to peer2 ==\n");
 
     env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
     let peer2_proj = format!("peer2_proj_{}", timestamp());
-    let (is_parent, _) = run_lnk(
-        LnkCmd::Clone(proj_urn, peer1_peer_id, peer2_proj.clone()),
-        peer2_home,
-        passphrase,
-    );
+    let cmd = LnkCmd::Clone(proj_urn, peer1_peer_id, peer2_proj.clone());
+    run_lnk!(&cmd, peer2_home, passphrase);
     if !is_parent {
         return;
     }
@@ -206,122 +167,125 @@ enum LnkCmd {
     IdPersonCreate(String),  // the associated string is "the person's name".
     IdLocalSet(String),      // the associated string is "urn".
     IdProjectCreate(String), // the associated string is "the project name".
-    Clone(String, String, String),   // the associated string is "urn", "peer_id", "path"
+    Clone(String, String, String), // the associated string is "urn", "peer_id", "path"
 }
 
-/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
-/// Return.0: true if this is the parent (i.e. test) process,
-///           false if this is the child (i.e. lnk) process.
-/// Return.1: an output that depends on the `cmd`.
-fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
-    let fork = Fork::from_ptmx().unwrap();
-    if let Some(mut parent) = fork.is_parent().ok() {
-        // Input the passphrase if necessary.
-        match cmd {
-            LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
-                parent.write_all(passphrase).unwrap();
-                println!("{}: wrote passphase", lnk_home);
-            },
-            _ => {},
-        }
-
-        // Print the output and decode them if necessary.
-        let buf_reader = BufReader::new(parent);
-        let mut output = String::new();
-        for line in buf_reader.lines() {
-            let line = line.unwrap();
-            println!("{}: {}", lnk_home, line);
-
-            match cmd {
-                LnkCmd::IdPersonCreate(ref _name) => {
-                    if line.find("\"urn\":").is_some() {
-                        output = line; // get the line with URN.
-                    }
-                },
-                LnkCmd::IdProjectCreate(ref _name) => {
-                    if line.find("\"urn\":").is_some() {
-                        output = line; // get the line with URN.
-                    }
-                },
-                LnkCmd::ProfileGet => {
-                    output = line; // get the last line for profile id.
-                },
-                LnkCmd::ProfilePeer => {
-                    output = line; // get the last line for peer id.
-                },
-                LnkCmd::Clone(ref _urn, ref _peer, ref _path) => {
-                    output = line;
+/// Runs a `lnk` command of `$cmd` using `$lnk_home` as the node home.
+/// Also support the passphrase input for commands that need it.
+#[macro_export]
+macro_rules! run_lnk {
+    ( $cmd:expr, $lnk_home:ident, $passphrase:ident ) => {{
+        let fork = Fork::from_ptmx().unwrap();
+        if let Some(mut parent) = fork.is_parent().ok() {
+            match $cmd {
+                LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
+                    // Input the passphrase if necessary.
+                    parent.write_all($passphrase).unwrap();
                 },
                 _ => {},
             }
+            process_lnk_output($lnk_home, &mut parent, $cmd)
+        } else {
+            start_lnk_cmd($lnk_home, $cmd);
+            return;
         }
+    }};
+}
 
-        (true, output)
-    } else {
-        // Child process is to run `lnk`.
-        let manifest_path = manifest_path();
-
-        // cargo run \
-        // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
-        // -p lnk -- "$@"
-        let mut lnk_cmd = Command::new("cargo");
-        lnk_cmd
-            .env("LNK_HOME", lnk_home)
-            .arg("run")
-            .arg("--manifest-path")
-            .arg(manifest_path)
-            .arg("-p")
-            .arg("lnk")
-            .arg("--");
-        let full_cmd = match cmd {
-            LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
-            LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
-            LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
-            LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
-            LnkCmd::IdPersonCreate(name) => {
-                let payload = json!({ "name": name });
-                lnk_cmd
-                    .arg("identities")
-                    .arg("person")
-                    .arg("create")
-                    .arg("new")
-                    .arg("--payload")
-                    .arg(payload.to_string())
+fn process_lnk_output(lnk_home: &str, lnk_process: &mut Master, cmd: &LnkCmd) -> String {
+    let buf_reader = BufReader::new(lnk_process);
+    let mut output = String::new();
+    for line in buf_reader.lines() {
+        let line = line.unwrap();
+
+        // Print the output and decode them if necessary.
+        println!("{}: {}", lnk_home, line);
+        match cmd {
+            LnkCmd::IdPersonCreate(ref _name) => {
+                if line.find("\"urn\":").is_some() {
+                    output = line; // get the line with URN.
+                }
             },
-            LnkCmd::IdLocalSet(urn) => lnk_cmd
-                .arg("identities")
-                .arg("local")
-                .arg("set")
-                .arg("--urn")
-                .arg(urn),
-            LnkCmd::IdProjectCreate(name) => {
-                let payload = json!({"name": name, "default_branch": "master"});
-                let project_path = format!("./{}", name);
-                lnk_cmd
-                    .arg("identities")
-                    .arg("project")
-                    .arg("create")
-                    .arg("new")
-                    .arg("--path")
-                    .arg(project_path)
-                    .arg("--payload")
-                    .arg(payload.to_string())
+            LnkCmd::IdProjectCreate(ref _name) => {
+                if line.find("\"urn\":").is_some() {
+                    output = line; // get the line with URN.
+                }
             },
-            LnkCmd::Clone(urn, peer_id, peer2_proj) => {
-                lnk_cmd
-                    .arg("clone")
-                    .arg("--urn")
-                    .arg(urn)
-                    .arg("--path")
-                    .arg(peer2_proj)
-                    .arg("--peer")
-                    .arg(peer_id)
+            LnkCmd::ProfileGet => {
+                output = line; // get the last line for profile id.
             },
-        };
-        full_cmd.status().expect("lnk cmd failed:");
-
-        (false, String::new())
+            LnkCmd::ProfilePeer => {
+                output = line; // get the last line for peer id.
+            },
+            LnkCmd::Clone(ref _urn, ref _peer, ref _path) => {
+                output = line;
+            },
+            _ => {},
+        }
     }
+
+    output
+}
+
+fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
+    let manifest_path = manifest_path();
+
+    // cargo run \
+    // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
+    // -p lnk -- "$@"
+    let mut lnk_cmd = Command::new("cargo");
+    lnk_cmd
+        .env("LNK_HOME", lnk_home)
+        .arg("run")
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk")
+        .arg("--");
+    let full_cmd = match cmd {
+        LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
+        LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
+        LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
+        LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
+        LnkCmd::IdPersonCreate(name) => {
+            let payload = json!({ "name": name });
+            lnk_cmd
+                .arg("identities")
+                .arg("person")
+                .arg("create")
+                .arg("new")
+                .arg("--payload")
+                .arg(payload.to_string())
+        },
+        LnkCmd::IdLocalSet(urn) => lnk_cmd
+            .arg("identities")
+            .arg("local")
+            .arg("set")
+            .arg("--urn")
+            .arg(urn),
+        LnkCmd::IdProjectCreate(name) => {
+            let payload = json!({"name": name, "default_branch": "master"});
+            let project_path = format!("./{}", name);
+            lnk_cmd
+                .arg("identities")
+                .arg("project")
+                .arg("create")
+                .arg("new")
+                .arg("--path")
+                .arg(project_path)
+                .arg("--payload")
+                .arg(payload.to_string())
+        },
+        LnkCmd::Clone(urn, peer_id, peer2_proj) => lnk_cmd
+            .arg("clone")
+            .arg("--urn")
+            .arg(urn)
+            .arg("--path")
+            .arg(peer2_proj)
+            .arg("--peer")
+            .arg(peer_id),
+    };
+    full_cmd.status().expect("lnk cmd failed:");
 }
 
 fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
@@ -376,26 +340,25 @@ fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
         .output()
         .expect("cargo build lnk-gitd failed");
 
-        Command::new("systemd-socket-activate")
-            .arg("-l")
-            .arg(port)
-            .arg("--fdname=ssh")
-            .arg("-E")
-            .arg("SSH_AUTH_SOCK")
-            .arg("-E")
-            .arg("RUST_BACKTRACE")
-            .arg(&exec_path)
-            .arg(lnk_home)
-            .arg("--linkd-rpc-socket")
-            .arg(rpc_socket)
-            .arg("--push-seeds")
-            .arg("--fetch-seeds")
-            .arg("--linger-timeout")
-            .arg("10000")
-            .spawn()
-            .expect("lnk-gitd failed to start");
-            println!("started lnk-gitd");
-
+    Command::new("systemd-socket-activate")
+        .arg("-l")
+        .arg(port)
+        .arg("--fdname=ssh")
+        .arg("-E")
+        .arg("SSH_AUTH_SOCK")
+        .arg("-E")
+        .arg("RUST_BACKTRACE")
+        .arg(&exec_path)
+        .arg(lnk_home)
+        .arg("--linkd-rpc-socket")
+        .arg(rpc_socket)
+        .arg("--push-seeds")
+        .arg("--fetch-seeds")
+        .arg("--linger-timeout")
+        .arg("10000")
+        .spawn()
+        .expect("lnk-gitd failed to start");
+    println!("started lnk-gitd");
 }
 
 /// Returns true if this is the parent process,
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v5 5/5] remove the use of systemd-socket-activate

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220825210251.61675-6-keepsimple@gmail.com>
In-Reply-To: <20220825210251.61675-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +10 -24

---
 bins/tests/integration_test.rs | 34 ++++++++++------------------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
index 43de968a..d245748d 100644
--- a/bins/tests/integration_test.rs
@@ -94,8 +94,9 @@ fn two_peers_and_a_seed() {
 
     println!("\n== Start the peer 1 gitd ==\n");
     let cmd = LnkCmd::ProfilePeer;
+    let gitd_addr = "127.0.0.1:9987";
     let peer1_peer_id = run_lnk!(&cmd, peer1_home, passphrase);
-    spawn_lnk_gitd(peer1_home, &manifest_path, &peer1_peer_id);
+    let mut gitd = spawn_lnk_gitd(peer1_home, &manifest_path, gitd_addr);
 
     println!("\n== Make some changes in the repo ==\n");
     env::set_current_dir(&peer1_proj).unwrap();
@@ -115,7 +116,7 @@ fn two_peers_and_a_seed() {
     println!("git-commit: {:?}", &output);
 
     println!("\n== Add the linkd remote to the repo ==\n");
-    let remote_url = format!("ssh://rad@127.0.0.1:9987/{}.git", &proj_urn);
+    let remote_url = format!("ssh://rad@{}/{}.git", gitd_addr, &proj_urn);
     Command::new("git")
         .arg("remote")
         .arg("add")
@@ -132,10 +133,6 @@ fn two_peers_and_a_seed() {
     }
 
     let peer1_last_commit = git_last_commit();
-    println!(
-        "\n== peer1 project last commit: {} ==\n",
-        &peer1_last_commit
-    );
 
     println!("\n== Clone to peer2 ==\n");
 
@@ -152,9 +149,10 @@ fn two_peers_and_a_seed() {
     println!("\n== peer1 proj last commit: {}", &peer1_last_commit);
     println!("\n== peer2 proj last commit: {}", &peer2_last_commit);
 
-    println!("\n== Cleanup: kill linkd (seed) ==\n");
+    println!("\n== Cleanup: kill linkd (seed) and gitd (peer1) ==\n");
 
     linkd.kill().ok();
+    gitd.kill().ok();
 
     assert_eq!(peer1_last_commit, peer2_last_commit);
 }
@@ -321,10 +319,7 @@ fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
     child
 }
 
-fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
-    let port = "9987";
-    let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
-    let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
+fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, addr: &str) -> Child {
     let target_dir = bins_target_dir();
     let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
 
@@ -340,25 +335,16 @@ fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
         .output()
         .expect("cargo build lnk-gitd failed");
 
-    Command::new("systemd-socket-activate")
-        .arg("-l")
-        .arg(port)
-        .arg("--fdname=ssh")
-        .arg("-E")
-        .arg("SSH_AUTH_SOCK")
-        .arg("-E")
-        .arg("RUST_BACKTRACE")
-        .arg(&exec_path)
+    let child = Command::new(&exec_path)
         .arg(lnk_home)
-        .arg("--linkd-rpc-socket")
-        .arg(rpc_socket)
         .arg("--push-seeds")
         .arg("--fetch-seeds")
-        .arg("--linger-timeout")
-        .arg("10000")
+        .arg("-a")
+        .arg(addr)
         .spawn()
         .expect("lnk-gitd failed to start");
     println!("started lnk-gitd");
+    child
 }
 
 /// Returns true if this is the parent process,
-- 
2.32.0 (Apple Git-132)

Re: [PATCH radicle-link v4 1/3] Add integration test for bins and fix git version checks

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqfCtP63Mc7hvH_QxRfi2v=Sy0dRvWLBZ+rRc4RhxThNeg@mail.gmail.com>
In-Reply-To: <YJO9BUK_HUlkBhSsrb4FQfkM3yfC8Luxcl5Au7xnrx7Em5N5-zRKq9zwieQCZukLbS6w7lAwaBQ1Xhu9x4JIq20oVJDPJgQcvVErW8SIdNo=@protonmail.ch> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Thu, Aug 25, 2022 at 9:10 AM <MissM_signed@protonmail.ch> wrote:
>
> So I am trying to get some collaboration on testing going between testing CLI type binaries across
>
> Is there a reason none of these could be used ?

I am open to use any of those if they are deemed necessary or helpful.
So far I've built the integration test
using the basic Rust / Cargo test facility and only one crate (`pty`).
I felt it provides a good baseline for
a testing of `bins`. We can always refactor the test code once we
decide to use some of the crates you
listed.

Another thing is that, as you probably mentioned I believe, a
well-documented test strategy is needed before
we adopt some framework-like crates, for example instra. If we already
have it, please let me know, I might
have missed something.  If we are in the process of defining the test
strategy, I'm open to the discussion.

>
> https://crates.io/crates/trycmd
> https://crates.io/crates/term-transcript
> out of date: https://github.com/rust-rspec/rspec
> https://crates.io/crates/trybuild
> https://nexte.st/book/leaky-tests.html
>
> compiletest seems to have detached yay https://github.com/Manishearth/compiletest-rs
> and ofc insta https://crates.io/crates/insta
> https://crates.io/crates/snapbox
> https://crates.io/crates/assert_cmd, https://crates.io/crates/assert_fs
> https://crates.io/crates/duct
> https://crates.io/crates/rexpect
> https://crates.io/crates/dir-diff
>
> I use insta a lot myself

Is there an example of how to use insta for testing some parts of
radicle-link? An example will be good if we want to adopt it going
forward.

>
> Rust idiomatic way is to use small crates from the ecosystem
>
> If we are still going ahead of using Rust to do integration tests.
>
> ------- Original Message -------
> On Friday, August 26th, 2022 at 1:44 AM, Alex Good <alex@memoryandthought.me> wrote:
>
>
> >
>
> >
>
> > On 24/08/22 10:41pm, Han Xu wrote:
> >
>
> > > diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
> > > +enum LnkCmd {
> > > + ProfileCreate,
> > > + ProfileGet,
> > > + ProfilePeer,
> > > + ProfileSshAdd,
> > > + IdPersonCreate(String), // the associated string is "the person's name".
> > > + IdLocalSet(String), // the associated string is "urn".
> > > + IdProjectCreate(String), // the associated string is "the project name".
> > > + Clone(String, String), // the associated string is "urn", "peer_id"
> > > +}
> > > +
> > > +/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
> > > +/// Return.0: true if this is the parent (i.e. test) process,
> > > +/// false if this is the child (i.e. lnk) process.
> > > +/// Return.1: an output that depends on the `cmd`.
> > > +fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
> >
>
> >
>
> > I still find this return and switch on the returned `bool` a bit
> > cumbersome. Is it worth using something like
> > https://docs.rs/pty-process/latest/pty_process/struct.Child.html
> > which would give us basically the same API as `std::process::Child`?
> >
>
> > Failing that we could write a little macro to remove all the `is_parent`
> > checks in the calling code.
> >
>
> > > +fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
> > > + let port = "9987";
> > > + let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
> > > + let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
> > > + let target_dir = bins_target_dir();
> > > + let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
> > > +
> > > + Command::new("cargo")
> > > + .arg("build")
> > > + .arg("--target-dir")
> > > + .arg(&target_dir)
> > > + .arg("--manifest-path")
> > > + .arg(manifest_path)
> > > + .arg("-p")
> > > + .arg("lnk-gitd")
> > > + .stdout(Stdio::inherit())
> > > + .output()
> > > + .expect("cargo build lnk-gitd failed");
> > > +
> > > + Command::new("systemd-socket-activate")
> > > + .arg("-l")
> > > + .arg(port)
> > > + .arg("--fdname=ssh")
> > > + .arg("-E")
> > > + .arg("SSH_AUTH_SOCK")
> > > + .arg("-E")
> > > + .arg("RUST_BACKTRACE")
> > > + .arg(&exec_path)
> > > + .arg(lnk_home)
> > > + .arg("--linkd-rpc-socket")
> > > + .arg(rpc_socket)
> > > + .arg("--push-seeds")
> > > + .arg("--fetch-seeds")
> > > + .arg("--linger-timeout")
> > > + .arg("10000")
> > > + .spawn()
> > > + .expect("lnk-gitd failed to start");
> >
>
> >
>
> > This is platform dependent (i.e. OSX doesn't have
> > `systemd-socket-activate`). We also don't really need to use the socket
> > activated version of this, we could just run `lnk_home --push-seeds --fetch-seeds --addr 127.0.0.1:<some port>` (i.e. no `--linger-timeout`)
> >
>
> > which will immediately run `lnk-gitd` and keep it running until it's
> > killed.

Re: [PATCH radicle-link v4 1/3] Add integration test for bins and fix git version checks

<MissM_signed@protonmail.ch>

Details

Message ID: <KE-jkC0ZYop48nD-rDUTrraG0-i4jkgdjcu8dQy1h2omkCWpE7OXbbd-i2ptWXHH9p1-gnOQwptB-NSM7v5tYwNlj8LH0_-jFHDYftsZ3BE=@protonmail.ch>
In-Reply-To: <CAEjGaqfCtP63Mc7hvH_QxRfi2v=Sy0dRvWLBZ+rRc4RhxThNeg@mail.gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

No, 


As a developer -

It is our job to choose the right tool for the job -

Re-using is one of the core competencies of ours -

I don't see even attempts to evaluate the options I gave you.

Don't expect anyone else to effectively do your work after for you.

Correcting mistakes is one but not even looking at the list and engaging in technical merits is another.

It is a massive stress having to clean up after someone when they were told of options to do it properly without extra effort.

We don't really need a strategy to do the right thing (tm) 


This fallacy goes two ways - 


Now we are adding code that should not exists without strategy -

We are now adding some low level code that is wholly redundant to test other low level code.. 


I mean what are we testing here.. the tests or the code ?

Like why are we dealing with pty's and the logistics of it ?

When e.g. trycmd - first on the list - does that in minimal manner ? 


Think of it like this:

   Every line of code is liability.

Just pick the right tool for the job (tm)

------- Original Message -------
On Friday, August 26th, 2022 at 7:15 AM, Han <keepsimple@gmail.com> wrote:


> 

> 

> On Thu, Aug 25, 2022 at 9:10 AM MissM_signed@protonmail.ch wrote:
> 

> > So I am trying to get some collaboration on testing going between testing CLI type binaries across
> > 

> > Is there a reason none of these could be used ?
> 

> 

> I am open to use any of those if they are deemed necessary or helpful.
> So far I've built the integration test
> using the basic Rust / Cargo test facility and only one crate (`pty`).
> I felt it provides a good baseline for
> a testing of `bins`. We can always refactor the test code once we
> decide to use some of the crates you
> listed.
> 

> Another thing is that, as you probably mentioned I believe, a
> well-documented test strategy is needed before
> we adopt some framework-like crates, for example instra. If we already
> have it, please let me know, I might
> have missed something. If we are in the process of defining the test
> strategy, I'm open to the discussion.
> 

> > https://crates.io/crates/trycmd
> > https://crates.io/crates/term-transcript
> > out of date: https://github.com/rust-rspec/rspec
> > https://crates.io/crates/trybuild
> > https://nexte.st/book/leaky-tests.html
> > 

> > compiletest seems to have detached yay https://github.com/Manishearth/compiletest-rs
> > and ofc insta https://crates.io/crates/insta
> > https://crates.io/crates/snapbox
> > https://crates.io/crates/assert_cmd, https://crates.io/crates/assert_fs
> > https://crates.io/crates/duct
> > https://crates.io/crates/rexpect
> > https://crates.io/crates/dir-diff
> > 

> > I use insta a lot myself
> 

> 

> Is there an example of how to use insta for testing some parts of
> radicle-link? An example will be good if we want to adopt it going
> forward.
> 

> > Rust idiomatic way is to use small crates from the ecosystem
> > 

> > If we are still going ahead of using Rust to do integration tests.
> > 

> > ------- Original Message -------
> > On Friday, August 26th, 2022 at 1:44 AM, Alex Good alex@memoryandthought.me wrote:
> > 

> > > On 24/08/22 10:41pm, Han Xu wrote:
> > 

> > > > diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
> > > > +enum LnkCmd {
> > > > + ProfileCreate,
> > > > + ProfileGet,
> > > > + ProfilePeer,
> > > > + ProfileSshAdd,
> > > > + IdPersonCreate(String), // the associated string is "the person's name".
> > > > + IdLocalSet(String), // the associated string is "urn".
> > > > + IdProjectCreate(String), // the associated string is "the project name".
> > > > + Clone(String, String), // the associated string is "urn", "peer_id"
> > > > +}
> > > > +
> > > > +/// Runs a `cmd` for `lnk_home`. Rebuilds `lnk` if necessary.
> > > > +/// Return.0: true if this is the parent (i.e. test) process,
> > > > +/// false if this is the child (i.e. lnk) process.
> > > > +/// Return.1: an output that depends on the `cmd`.
> > > > +fn run_lnk(cmd: LnkCmd, lnk_home: &str, passphrase: &[u8]) -> (bool, String) {
> > 

> > > I still find this return and switch on the returned `bool` a bit
> > > cumbersome. Is it worth using something like
> > > https://docs.rs/pty-process/latest/pty_process/struct.Child.html
> > > which would give us basically the same API as `std::process::Child`?
> > 

> > > Failing that we could write a little macro to remove all the `is_parent`
> > > checks in the calling code.
> > 

> > > > +fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, peer_id: &str) {
> > > > + let port = "9987";
> > > > + let xdg_runtime_dir = env!("XDG_RUNTIME_DIR");
> > > > + let rpc_socket = format!("{}/link-peer-{}-rpc.socket", xdg_runtime_dir, peer_id);
> > > > + let target_dir = bins_target_dir();
> > > > + let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
> > > > +
> > > > + Command::new("cargo")
> > > > + .arg("build")
> > > > + .arg("--target-dir")
> > > > + .arg(&target_dir)
> > > > + .arg("--manifest-path")
> > > > + .arg(manifest_path)
> > > > + .arg("-p")
> > > > + .arg("lnk-gitd")
> > > > + .stdout(Stdio::inherit())
> > > > + .output()
> > > > + .expect("cargo build lnk-gitd failed");
> > > > +
> > > > + Command::new("systemd-socket-activate")
> > > > + .arg("-l")
> > > > + .arg(port)
> > > > + .arg("--fdname=ssh")
> > > > + .arg("-E")
> > > > + .arg("SSH_AUTH_SOCK")
> > > > + .arg("-E")
> > > > + .arg("RUST_BACKTRACE")
> > > > + .arg(&exec_path)
> > > > + .arg(lnk_home)
> > > > + .arg("--linkd-rpc-socket")
> > > > + .arg(rpc_socket)
> > > > + .arg("--push-seeds")
> > > > + .arg("--fetch-seeds")
> > > > + .arg("--linger-timeout")
> > > > + .arg("10000")
> > > > + .spawn()
> > > > + .expect("lnk-gitd failed to start");
> > 

> > > This is platform dependent (i.e. OSX doesn't have
> > > `systemd-socket-activate`). We also don't really need to use the socket
> > > activated version of this, we could just run `lnk_home --push-seeds --fetch-seeds --addr 127.0.0.1:<some port>` (i.e. no `--linger-timeout`)
> > 

> > > which will immediately run `lnk-gitd` and keep it running until it's
> > > killed.

Re: [PATCH radicle-link v4 1/3] Add integration test for bins and fix git version checks

Alex Good <alex@memoryandthought.me>

Details

Message ID: <CAH_DpYRTUQUfUWrbqUFgY67wvAjfv36tQVjy1bnYqPJiH7U5hA@mail.gmail.com>
In-Reply-To: <KE-jkC0ZYop48nD-rDUTrraG0-i4jkgdjcu8dQy1h2omkCWpE7OXbbd-i2ptWXHH9p1-gnOQwptB-NSM7v5tYwNlj8LH0_-jFHDYftsZ3BE=@protonmail.ch> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On 26/08/22 02:49am, MissM_signed@protonmail.ch wrote:
> No,
>
>
> As a developer -
>
> It is our job to choose the right tool for the job -
>
> Re-using is one of the core competencies of ours -
>
> I don't see even attempts to evaluate the options I gave you.

To me that seems like work that you should be doing. Both Han and I have
responded to a previous message you sent about using a library for this.
We both took the time to outline our concerns in detail, for some reason
you decided not to respond.

It seems then that we have a difference of opinion on what should be
done here. We're all collaborators here, and you seem to have the most
experience with these libraries, you're also the person most interested
in using these libraries, as such you seem best placed to present the
case for them - I would be interested to hear it.

>
> Don't expect anyone else to effectively do your work after for you.

This is an extremely uncharitable take. Everyone here is working hard in
good faith. Han is on the fith reroll of this patch and has been very
willing to accomodate feedback - he's clearly not expecting other people
to do his work.

>
> Correcting mistakes is one but not even looking at the list and engaging in technical merits is another.
>
> It is a massive stress having to clean up after someone when they were told of options to do it properly without extra effort.
>
> We don't really need a strategy to do the right thing (tm)
>
>
> This fallacy goes two ways -
>
>
> Now we are adding code that should not exists without strategy -
>
> We are now adding some low level code that is wholly redundant to test other low level code..
>
>
> I mean what are we testing here.. the tests or the code ?
>
> Like why are we dealing with pty's and the logistics of it ?
>
> When e.g. trycmd - first on the list - does that in minimal manner ?

`trycmd` doesn't seem minimal to me, it requires rewriting your tests as
literate markdown files. This is quite dramatic - maybe it's good, maybe
it's bad, but it's certainly not an obvious decision. As I said, you
seem to be more familiar with all these libraries, I would be keen to see
what you think the code should look like and what you think the
advantages and disadvantages are.

Re: [PATCH radicle-link v5 0/5] Add integration test for bins and fix bugs found

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CMIR1M1UGAZY.11OMO37UUIDXM@haptop>
In-Reply-To: <20220825210251.61675-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +100 -65

I have some stylistic suggestions for test file below.

I think you can also squash the changes to that same file into a
single commit. So, the patch should look like the addition of the
integration_test.rs file and the fix to the signed_refs update.

---
 bins/tests/integration_test.rs | 165 ++++++++++++++++++++-------------
 1 file changed, 100 insertions(+), 65 deletions(-)

diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
index d245748d..91d4d423 100644
--- a/bins/tests/integration_test.rs
@@ -34,59 +34,79 @@ fn two_peers_and_a_seed() {
     let seed_home = "/tmp/seed-home";
     let passphrase = b"play\n";
 
-    println!("\n== create lnk homes for two peers and one seed ==\n");
-    let cmd = LnkCmd::ProfileCreate;
-    run_lnk!(&cmd, peer1_home, passphrase);
-    run_lnk!(&cmd, peer2_home, passphrase);
-    run_lnk!(&cmd, seed_home, passphrase);
+    {
+        println!("\n== create lnk homes for two peers and one seed ==\n");
+        let cmd = LnkCmd::ProfileCreate;
+        run_lnk!(&cmd, peer1_home, passphrase);
+        run_lnk!(&cmd, peer2_home, passphrase);
+        run_lnk!(&cmd, seed_home, passphrase);
+    }
 
-    println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
-    let cmd = LnkCmd::ProfileSshAdd;
-    run_lnk!(&cmd, peer1_home, passphrase);
-    run_lnk!(&cmd, peer2_home, passphrase);
-    run_lnk!(&cmd, seed_home, passphrase);
-
-    println!("\n== Creating local link 1 identity ==\n");
-    let peer1_name = "sockpuppet1".to_string();
-    let cmd = LnkCmd::IdPersonCreate(peer1_name);
-    let output = run_lnk!(&cmd, peer1_home, passphrase);
-    let v: Value = serde_json::from_str(&output).unwrap();
-    let urn1 = v["urn"].as_str().unwrap().to_string();
-    let cmd = LnkCmd::IdLocalSet(urn1);
-    run_lnk!(&cmd, peer1_home, passphrase);
-
-    println!("\n== Creating local link 2 identity ==\n");
-    let peer2_name = "sockpuppet2".to_string();
-    let cmd = LnkCmd::IdPersonCreate(peer2_name);
-    let output = run_lnk!(&cmd, peer2_home, passphrase);
-    let v: Value = serde_json::from_str(&output).unwrap();
-    let urn2 = v["urn"].as_str().unwrap().to_string();
-    let cmd = LnkCmd::IdLocalSet(urn2);
-    run_lnk!(&cmd, peer2_home, passphrase);
+    {
+        println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
+        let cmd = LnkCmd::ProfileSshAdd;
+        run_lnk!(&cmd, peer1_home, passphrase);
+        run_lnk!(&cmd, peer2_home, passphrase);
+        run_lnk!(&cmd, seed_home, passphrase);
+    }
 
-    println!("\n== Create a local repository ==\n");
-    let peer1_proj = format!("peer1_proj_{}", timestamp());
-    let cmd = LnkCmd::IdProjectCreate(peer1_proj.clone());
-    let output = run_lnk!(&cmd, peer1_home, passphrase);
-    let v: Value = serde_json::from_str(&output).unwrap();
-    let proj_urn = v["urn"].as_str().unwrap().to_string();
-    println!("our project URN: {}", &proj_urn);
+    {
+        println!("\n== Creating local link 1 identity ==\n");
+        let name = "sockpuppet1".to_string();
+        let cmd = LnkCmd::IdPersonCreate { name };
+        let output = run_lnk!(&cmd, peer1_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let urn = v["urn"].as_str().unwrap().to_string();
+        let cmd = LnkCmd::IdLocalSet { urn: urn };
+        run_lnk!(&cmd, peer1_home, passphrase);
+    }
 
-    println!("\n== Add the seed to the local peer seed configs ==\n");
-    let cmd = LnkCmd::ProfilePeer;
-    let seed_peer_id = run_lnk!(&cmd, seed_home, passphrase);
-    let seed_endpoint = format!("{}@127.0.0.1:8799", &seed_peer_id);
+    {
+        println!("\n== Creating local link 2 identity ==\n");
+        let name = "sockpuppet2".to_string();
+        let cmd = LnkCmd::IdPersonCreate { name };
+        let output = run_lnk!(&cmd, peer2_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let urn = v["urn"].as_str().unwrap().to_string();
+        let cmd = LnkCmd::IdLocalSet { urn };
+        run_lnk!(&cmd, peer2_home, passphrase);
+    }
+
+    let urn = {
+        println!("\n== Create a local repository ==\n");
+        let peer1_proj = format!("peer1_proj_{}", timestamp());
+        let cmd = LnkCmd::IdProjectCreate {
+            name: peer1_proj.clone(),
+        };
+        let output = run_lnk!(&cmd, peer1_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let proj_urn = v["urn"].as_str().unwrap().to_string();
+        println!("our project URN: {}", &proj_urn);
+        proj_urn
+    };
 
-    let cmd = LnkCmd::ProfileGet;
-    let peer1_profile = run_lnk!(&cmd, peer1_home, passphrase);
-    let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
-    let mut peer1_f = File::create(peer1_seed).unwrap();
-    peer1_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    let seed_endpoint = {
+        println!("\n== Add the seed to the local peer seed configs ==\n");
+        let cmd = LnkCmd::ProfilePeer;
+        let seed_peer_id = run_lnk!(&cmd, seed_home, passphrase);
+        format!("{}@127.0.0.1:8799", &seed_peer_id)
+    };
 
-    let peer2_profile = run_lnk!(&cmd, peer2_home, passphrase);
-    let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
-    let mut peer2_f = File::create(peer2_seed).unwrap();
-    peer2_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    {
+        let cmd = LnkCmd::ProfileGet;
+        let peer1_profile = run_lnk!(&cmd, peer1_home, passphrase);
+        let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
+        let mut peer1_f = File::create(peer1_seed).unwrap();
+        peer1_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    }
+
+    {
+        let cmd = LnkCmd::ProfileGet;
+        let peer2_profile = run_lnk!(&cmd, peer2_home, passphrase);
+        let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
+        let mut peer2_f = File::create(peer2_seed).unwrap();
+        peer2_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    }
 
     println!("\n== Start the seed linkd ==\n");
     let manifest_path = manifest_path();
@@ -99,7 +119,7 @@ fn two_peers_and_a_seed() {
     let mut gitd = spawn_lnk_gitd(peer1_home, &manifest_path, gitd_addr);
 
     println!("\n== Make some changes in the repo ==\n");
-    env::set_current_dir(&peer1_proj).unwrap();
+    env::set_current_dir(&urn).unwrap();
     let mut test_file = File::create("test").unwrap();
     test_file.write_all(b"test").unwrap();
     Command::new("git")
@@ -116,7 +136,7 @@ fn two_peers_and_a_seed() {
     println!("git-commit: {:?}", &output);
 
     println!("\n== Add the linkd remote to the repo ==\n");
-    let remote_url = format!("ssh://rad@{}/{}.git", gitd_addr, &proj_urn);
+    let remote_url = format!("ssh://rad@{}/{}.git", gitd_addr, &urn);
     Command::new("git")
         .arg("remote")
         .arg("add")
@@ -138,11 +158,12 @@ fn two_peers_and_a_seed() {
 
     env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
     let peer2_proj = format!("peer2_proj_{}", timestamp());
-    let cmd = LnkCmd::Clone(proj_urn, peer1_peer_id, peer2_proj.clone());
+    let cmd = LnkCmd::Clone {
+        urn,
+        peer_id: peer1_peer_id,
+        path: peer2_proj.clone(),
+    };
     run_lnk!(&cmd, peer2_home, passphrase);
-    if !is_parent {
-        return;
-    }
 
     env::set_current_dir(peer2_proj).unwrap();
     let peer2_last_commit = git_last_commit();
@@ -162,10 +183,20 @@ enum LnkCmd {
     ProfileGet,
     ProfilePeer,
     ProfileSshAdd,
-    IdPersonCreate(String),  // the associated string is "the person's name".
-    IdLocalSet(String),      // the associated string is "urn".
-    IdProjectCreate(String), // the associated string is "the project name".
-    Clone(String, String, String), // the associated string is "urn", "peer_id", "path"
+    IdPersonCreate {
+        name: String,
+    },
+    IdLocalSet {
+        urn: String,
+    },
+    IdProjectCreate {
+        name: String,
+    },
+    Clone {
+        urn: String,
+        peer_id: String,
+        path: String,
+    },
 }
 
 /// Runs a `lnk` command of `$cmd` using `$lnk_home` as the node home.
@@ -199,12 +230,12 @@ fn process_lnk_output(lnk_home: &str, lnk_process: &mut Master, cmd: &LnkCmd) ->
         // Print the output and decode them if necessary.
         println!("{}: {}", lnk_home, line);
         match cmd {
-            LnkCmd::IdPersonCreate(ref _name) => {
+            LnkCmd::IdPersonCreate { .. } => {
                 if line.find("\"urn\":").is_some() {
                     output = line; // get the line with URN.
                 }
             },
-            LnkCmd::IdProjectCreate(ref _name) => {
+            LnkCmd::IdProjectCreate { .. } => {
                 if line.find("\"urn\":").is_some() {
                     output = line; // get the line with URN.
                 }
@@ -215,7 +246,7 @@ fn process_lnk_output(lnk_home: &str, lnk_process: &mut Master, cmd: &LnkCmd) ->
             LnkCmd::ProfilePeer => {
                 output = line; // get the last line for peer id.
             },
-            LnkCmd::Clone(ref _urn, ref _peer, ref _path) => {
+            LnkCmd::Clone { .. } => {
                 output = line;
             },
             _ => {},
@@ -245,7 +276,7 @@ fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
         LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
         LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
         LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
-        LnkCmd::IdPersonCreate(name) => {
+        LnkCmd::IdPersonCreate { name } => {
             let payload = json!({ "name": name });
             lnk_cmd
                 .arg("identities")
@@ -255,13 +286,13 @@ fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
                 .arg("--payload")
                 .arg(payload.to_string())
         },
-        LnkCmd::IdLocalSet(urn) => lnk_cmd
+        LnkCmd::IdLocalSet { urn } => lnk_cmd
             .arg("identities")
             .arg("local")
             .arg("set")
             .arg("--urn")
             .arg(urn),
-        LnkCmd::IdProjectCreate(name) => {
+        LnkCmd::IdProjectCreate { name } => {
             let payload = json!({"name": name, "default_branch": "master"});
             let project_path = format!("./{}", name);
             lnk_cmd
@@ -274,7 +305,11 @@ fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
                 .arg("--payload")
                 .arg(payload.to_string())
         },
-        LnkCmd::Clone(urn, peer_id, peer2_proj) => lnk_cmd
+        LnkCmd::Clone {
+            urn,
+            peer_id,
+            path: peer2_proj,
+        } => lnk_cmd
             .arg("clone")
             .arg("--urn")
             .arg(urn)
-- 
2.31.1

[PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220830045938.93431-1-keepsimple@gmail.com>
In-Reply-To: <20220825210251.61675-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

This is the 6th version (re-rolling) patch to add an integration test for `bins` package.

It is built on top of version 5 patch and addressed review comments:
 - refactor the code styling.
 - squash the patch into two commits only.

The test case scenario follows the steps in https://github.com/alexjg/linkd-playground ,
except it is in Rust now and we can just run `cargo test`. To see a more helpful output,
please run `cargo test -- --nocapture` from the `bins/tests` directory.

We also found two bugs along the way: 

1. Git version checks are off. In our testing, it shows that
git 2.25.1 on Ubuntu 20.04 and git 2.30.2 on Debian 11 do not need the workaround that
adds "namespaces" manually. Hence fixing the git version checked against.

2. `request_pull` to the seed fails to pull the latest commit. The reason being that
the signed_refs for `heads/master` is not the latest. A fix is provided.


Han Xu (2):
  Add integration test for bins and fix git version checks
  fix post_receive to send updated refs in request_pull

 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 456 +++++++++++++++++++++++++++++++++
 cli/gitd-lib/src/hooks.rs      |  19 +-
 link-git/src/protocol/fetch.rs |   5 +-
 link-git/src/protocol/ls.rs    |   5 +-
 7 files changed, 518 insertions(+), 10 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v6 1/2] Add integration test for bins and fix git version checks

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220830045938.93431-2-keepsimple@gmail.com>
In-Reply-To: <20220830045938.93431-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +507 -2

---
 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 456 +++++++++++++++++++++++++++++++++
 link-git/src/protocol/fetch.rs |   5 +-
 link-git/src/protocol/ls.rs    |   5 +-
 6 files changed, 507 insertions(+), 2 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

diff --git a/bins/Cargo.lock b/bins/Cargo.lock
index d5756144..4d0ea413 100644
--- a/bins/Cargo.lock
@@ -838,6 +838,17 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "errno"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e2b2decb0484e15560df3210cf0d78654bb0864b2c138977c07e377a1bae0e2"
+dependencies = [
+ "kernel32-sys",
+ "libc",
+ "winapi 0.2.8",
+]
+
 [[package]]
 name = "event-listener"
 version = "2.5.2"
@@ -2945,6 +2956,16 @@ dependencies = [
  "human_format",
 ]
 
+[[package]]
+name = "pty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50f3d255966981eb4e4c5df3e983e6f7d163221f547406d83b6a460ff5c5ee8"
+dependencies = [
+ "errno",
+ "libc",
+]
+
 [[package]]
 name = "quanta"
 version = "0.4.1"
@@ -3567,6 +3588,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "tests"
+version = "0.1.0"
+dependencies = [
+ "pty",
+ "serde_json",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
diff --git a/bins/Cargo.toml b/bins/Cargo.toml
index 3d1786f4..99b3c69f 100644
--- a/bins/Cargo.toml
@@ -5,4 +5,5 @@ members = [
   "lnk-gitd",
   "lnk-identities-dev",
   "lnk-profile-dev",
+  "tests",
 ]
diff --git a/bins/tests/Cargo.toml b/bins/tests/Cargo.toml
new file mode 100644
index 00000000..3ae82769
--- /dev/null
@@ -0,0 +1,13 @@
+[package]
+name = "tests"
+version = "0.1.0"
+authors = ["Han Xu <keepsimple@gmail.com>"]
+edition = "2021"
+
+[dev-dependencies]
+pty = "0.2"
+serde_json = "1.0"
+
+[[test]]
+name = "integration_test"
+path = "integration_test.rs"
diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
new file mode 100644
index 00000000..2d2a4a92
--- /dev/null
@@ -0,0 +1,456 @@
+// Copyright © 2022 The Radicle Link Contributors
+//
+// This file is part of radicle-link, distributed under the GPLv3 with Radicle
+// Linking Exception. For full terms see the included LICENSE file.
+
+//! Integration test to exercise the programs in `bins`.
+
+use pty::fork::*;
+use serde_json::{json, Value};
+use std::{
+    env,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    process::{Child, Command, Stdio},
+    thread,
+    time::{Duration, SystemTime},
+};
+
+/// This test is inspired by https://github.com/alexjg/linkd-playground
+///
+/// Tests a typical scenario: there are two peer nodes and one seed node.
+/// The main steps are:
+///   - Setup a profile for each node in tmp home directories.
+///   - Setup SSH keys for each profile.
+///   - Create identities.
+///   - Create a local repo for peer 1.
+///   - Start `linkd` for the seed, and `lnk-gitd` for peer 1.
+///   - Push peer 1 repo to its monorepo and to the seed.
+///   - Clone the peer 1 repo to peer 2 via seed.
+///
+/// This test only works in Linux at this moment.
+#[test]
+fn two_peers_and_a_seed() {
+    let peer1_home = "/tmp/link-local-1";
+    let peer2_home = "/tmp/link-local-2";
+    let seed_home = "/tmp/seed-home";
+    let passphrase = b"play\n";
+
+    {
+        println!("\n== create lnk homes for two peers and one seed ==\n");
+        let cmd = LnkCmd::ProfileCreate;
+        run_lnk!(&cmd, peer1_home, passphrase);
+        run_lnk!(&cmd, peer2_home, passphrase);
+        run_lnk!(&cmd, seed_home, passphrase);
+    }
+
+    {
+        println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
+        let cmd = LnkCmd::ProfileSshAdd;
+        run_lnk!(&cmd, peer1_home, passphrase);
+        run_lnk!(&cmd, peer2_home, passphrase);
+        run_lnk!(&cmd, seed_home, passphrase);
+    }
+
+    {
+        println!("\n== Creating local link 1 identity ==\n");
+        let peer1_name = "sockpuppet1".to_string();
+        let cmd = LnkCmd::IdPersonCreate { name: peer1_name };
+        let output = run_lnk!(&cmd, peer1_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let urn = v["urn"].as_str().unwrap().to_string();
+        let cmd = LnkCmd::IdLocalSet { urn };
+        run_lnk!(&cmd, peer1_home, passphrase);
+    }
+
+    {
+        println!("\n== Creating local link 2 identity ==\n");
+        let peer2_name = "sockpuppet2".to_string();
+        let cmd = LnkCmd::IdPersonCreate { name: peer2_name };
+        let output = run_lnk!(&cmd, peer2_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let urn = v["urn"].as_str().unwrap().to_string();
+        let cmd = LnkCmd::IdLocalSet { urn };
+        run_lnk!(&cmd, peer2_home, passphrase);
+    }
+
+    let peer1_proj_dir = format!("peer1_proj_{}", timestamp());
+    let peer1_proj_urn = {
+        println!("\n== Create a local repository for peer1 ==\n");
+        let cmd = LnkCmd::IdProjectCreate {
+            name: peer1_proj_dir.clone(),
+        };
+        let output = run_lnk!(&cmd, peer1_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let proj_urn = v["urn"].as_str().unwrap().to_string();
+        println!("our project URN: {}", &proj_urn);
+        proj_urn
+    };
+
+    println!("\n== Add the seed to the local peer seed configs ==\n");
+    let seed_endpoint = {
+        let cmd = LnkCmd::ProfilePeer;
+        let seed_peer_id = run_lnk!(&cmd, seed_home, passphrase);
+        format!("{}@127.0.0.1:8799", &seed_peer_id)
+    };
+
+    {
+        // Create seed file for peer1
+        let cmd = LnkCmd::ProfileGet;
+        let peer1_profile = run_lnk!(&cmd, peer1_home, passphrase);
+        let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
+        let mut peer1_seed_f = File::create(peer1_seed).unwrap();
+        peer1_seed_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    }
+
+    {
+        // Create seed file for peer2
+        let cmd = LnkCmd::ProfileGet;
+        let peer2_profile = run_lnk!(&cmd, peer2_home, passphrase);
+        let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
+        let mut peer2_seed_f = File::create(peer2_seed).unwrap();
+        peer2_seed_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    }
+
+    println!("\n== Start the seed linkd ==\n");
+    let manifest_path = manifest_path();
+    let mut linkd = spawn_linkd(seed_home, &manifest_path);
+
+    println!("\n== Start the peer 1 gitd ==\n");
+    let cmd = LnkCmd::ProfilePeer;
+    let gitd_addr = "127.0.0.1:9987";
+    let peer1_peer_id = run_lnk!(&cmd, peer1_home, passphrase);
+    let mut gitd = spawn_lnk_gitd(peer1_home, &manifest_path, gitd_addr);
+
+    println!("\n== Make some changes in the repo: add and commit a test file ==\n");
+    env::set_current_dir(&peer1_proj_dir).unwrap();
+    let mut test_file = File::create("test").unwrap();
+    test_file.write_all(b"test").unwrap();
+    Command::new("git")
+        .arg("add")
+        .arg("test")
+        .output()
+        .expect("failed to do git add");
+    let output = Command::new("git")
+        .arg("commit")
+        .arg("-m")
+        .arg("test commit")
+        .output()
+        .expect("failed to do git commit");
+    println!("git-commit: {:?}", &output);
+
+    println!("\n== Add the linkd remote to the repo ==\n");
+    let remote_url = format!("ssh://rad@{}/{}.git", gitd_addr, &peer1_proj_urn);
+    Command::new("git")
+        .arg("remote")
+        .arg("add")
+        .arg("linkd")
+        .arg(remote_url)
+        .output()
+        .expect("failed to do git remote add");
+
+    clean_up_known_hosts();
+
+    if run_git_push_in_child_process() {
+        // The child process is done with git push.
+        return;
+    }
+
+    let peer1_last_commit = git_last_commit();
+
+    println!("\n== Clone to peer2 ==\n");
+
+    env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
+    let peer2_proj = format!("peer2_proj_{}", timestamp());
+    let cmd = LnkCmd::Clone {
+        urn: peer1_proj_urn,
+        peer_id: peer1_peer_id,
+        path: peer2_proj.clone(),
+    };
+    run_lnk!(&cmd, peer2_home, passphrase);
+
+    env::set_current_dir(peer2_proj).unwrap();
+    let peer2_last_commit = git_last_commit();
+    println!("\n== peer1 proj last commit: {}", &peer1_last_commit);
+    println!("\n== peer2 proj last commit: {}", &peer2_last_commit);
+
+    println!("\n== Cleanup: kill linkd (seed) and gitd (peer1) ==\n");
+
+    linkd.kill().ok();
+    gitd.kill().ok();
+
+    assert_eq!(peer1_last_commit, peer2_last_commit);
+}
+
+enum LnkCmd {
+    ProfileCreate,
+    ProfileGet,
+    ProfilePeer,
+    ProfileSshAdd,
+    IdPersonCreate {
+        name: String,
+    },
+    IdLocalSet {
+        urn: String,
+    },
+    IdProjectCreate {
+        name: String,
+    },
+    Clone {
+        urn: String,
+        peer_id: String,
+        path: String,
+    },
+}
+
+/// Runs a `lnk` command of `$cmd` using `$lnk_home` as the node home.
+/// Also support the passphrase input for commands that need it.
+#[macro_export]
+macro_rules! run_lnk {
+    ( $cmd:expr, $lnk_home:ident, $passphrase:ident ) => {{
+        let fork = Fork::from_ptmx().unwrap();
+        if let Some(mut parent) = fork.is_parent().ok() {
+            match $cmd {
+                LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
+                    // Input the passphrase if necessary.
+                    parent.write_all($passphrase).unwrap();
+                },
+                _ => {},
+            }
+            process_lnk_output($lnk_home, &mut parent, $cmd)
+        } else {
+            start_lnk_cmd($lnk_home, $cmd);
+            return;
+        }
+    }};
+}
+
+fn process_lnk_output(lnk_home: &str, lnk_process: &mut Master, cmd: &LnkCmd) -> String {
+    let buf_reader = BufReader::new(lnk_process);
+    let mut output = String::new();
+    for line in buf_reader.lines() {
+        let line = line.unwrap();
+
+        // Print the output and decode them if necessary.
+        println!("{}: {}", lnk_home, line);
+        match cmd {
+            LnkCmd::IdPersonCreate { name: _ } => {
+                if line.find("\"urn\":").is_some() {
+                    output = line; // get the line with URN.
+                }
+            },
+            LnkCmd::IdProjectCreate { name: _ } => {
+                if line.find("\"urn\":").is_some() {
+                    output = line; // get the line with URN.
+                }
+            },
+            LnkCmd::ProfileGet => {
+                output = line; // get the last line for profile id.
+            },
+            LnkCmd::ProfilePeer => {
+                output = line; // get the last line for peer id.
+            },
+            LnkCmd::Clone {
+                urn: _,
+                peer_id: _,
+                path: _,
+            } => {
+                output = line;
+            },
+            _ => {},
+        }
+    }
+
+    output
+}
+
+fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
+    let manifest_path = manifest_path();
+
+    // cargo run \
+    // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
+    // -p lnk -- "$@"
+    let mut lnk_cmd = Command::new("cargo");
+    lnk_cmd
+        .env("LNK_HOME", lnk_home)
+        .arg("run")
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk")
+        .arg("--");
+    let full_cmd = match cmd {
+        LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
+        LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
+        LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
+        LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
+        LnkCmd::IdPersonCreate { name } => {
+            let payload = json!({ "name": name });
+            lnk_cmd
+                .arg("identities")
+                .arg("person")
+                .arg("create")
+                .arg("new")
+                .arg("--payload")
+                .arg(payload.to_string())
+        },
+        LnkCmd::IdLocalSet { urn } => lnk_cmd
+            .arg("identities")
+            .arg("local")
+            .arg("set")
+            .arg("--urn")
+            .arg(urn),
+        LnkCmd::IdProjectCreate { name } => {
+            let payload = json!({"name": name, "default_branch": "master"});
+            let project_path = format!("./{}", name);
+            lnk_cmd
+                .arg("identities")
+                .arg("project")
+                .arg("create")
+                .arg("new")
+                .arg("--path")
+                .arg(project_path)
+                .arg("--payload")
+                .arg(payload.to_string())
+        },
+        LnkCmd::Clone { urn, peer_id, path } => lnk_cmd
+            .arg("clone")
+            .arg("--urn")
+            .arg(urn)
+            .arg("--path")
+            .arg(path)
+            .arg("--peer")
+            .arg(peer_id),
+    };
+    full_cmd.status().expect("lnk cmd failed:");
+}
+
+fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
+    let log_name = format!("linkd_{}.log", &timestamp());
+    let log_file = File::create(&log_name).unwrap();
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/linkd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("linkd")
+        .output()
+        .expect("cargo build linkd failed");
+
+    let child = Command::new(&exec_path)
+        .env("RUST_BACKTRACE", "1")
+        .arg("--lnk-home")
+        .arg(lnk_home)
+        .arg("--track")
+        .arg("everything")
+        .arg("--protocol-listen")
+        .arg("127.0.0.1:8799")
+        .stdout(Stdio::from(log_file))
+        .spawn()
+        .expect("linkd failed to start");
+    println!("linkd stdout redirected to {}", &log_name);
+    thread::sleep(Duration::from_secs(1));
+    child
+}
+
+fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, addr: &str) -> Child {
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk-gitd")
+        .stdout(Stdio::inherit())
+        .output()
+        .expect("cargo build lnk-gitd failed");
+
+    let child = Command::new(&exec_path)
+        .arg(lnk_home)
+        .arg("--push-seeds")
+        .arg("--fetch-seeds")
+        .arg("-a")
+        .arg(addr)
+        .spawn()
+        .expect("lnk-gitd failed to start");
+    println!("started lnk-gitd");
+    child
+}
+
+/// Returns true if runs in the forked child process for git push.
+/// returns false if runs in the parent process.
+fn run_git_push_in_child_process() -> bool {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        let yes = b"yes\n";
+        let buf_reader = BufReader::new(parent);
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("git-push: {}", line);
+            if line.find("key fingerprint").is_some() {
+                parent.write_all(yes).unwrap();
+            }
+        }
+
+        false // This is not the child process.
+    } else {
+        Command::new("git")
+            .arg("push")
+            .arg("linkd")
+            .status()
+            .expect("failed to do git push");
+        true // This is the child process.
+    }
+}
+
+fn git_last_commit() -> String {
+    let output = Command::new("git")
+        .arg("rev-parse")
+        .arg("HEAD")
+        .output()
+        .expect("failed to run git rev-parse");
+    String::from_utf8_lossy(&output.stdout).to_string()
+}
+
+/// Returns UNIX_TIME in millis.
+fn timestamp() -> u128 {
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap();
+    now.as_millis()
+}
+
+/// Returns the full path of `bins` manifest file.
+fn manifest_path() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/Cargo.toml", package_dir.strip_suffix("/tests").unwrap())
+}
+
+/// Returns the full path of `bins/target`.
+fn bins_target_dir() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/target", package_dir.strip_suffix("/tests").unwrap())
+}
+
+fn clean_up_known_hosts() {
+    // ssh-keygen -f "/home/pi/.ssh/known_hosts" -R "[127.0.0.1]:9987"
+    let home_dir = env!("HOME");
+    let known_hosts = format!("{}/.ssh/known_hosts", &home_dir);
+    let output = Command::new("ssh-keygen")
+        .arg("-f")
+        .arg(known_hosts)
+        .arg("-R")
+        .arg("[127.0.0.1]:9987")
+        .output()
+        .expect("failed to do ssh-keygen");
+    println!("ssh-keygen: {:?}", &output);
+}
diff --git a/link-git/src/protocol/fetch.rs b/link-git/src/protocol/fetch.rs
index a5a73065..98b92b51 100644
--- a/link-git/src/protocol/fetch.rs
+++ b/link-git/src/protocol/fetch.rs
@@ -38,8 +38,11 @@ use super::{packwriter::PackWriter, remote_git_version, transport};
 //
 // cf. https://lore.kernel.org/git/CD2XNXHACAXS.13J6JTWZPO1JA@schmidt/
 // Fixed in `git.git` 1ab13eb, which should land in 2.34
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace_want_ref(caps: &client::Capabilities) -> bool {
-    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.33.0").unwrap());
+    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version <= *FIXED_AFTER)
diff --git a/link-git/src/protocol/ls.rs b/link-git/src/protocol/ls.rs
index b3516454..de95b4a3 100644
--- a/link-git/src/protocol/ls.rs
+++ b/link-git/src/protocol/ls.rs
@@ -22,9 +22,12 @@ use super::{remote_git_version, transport};
 // Work around `git-upload-pack` not handling namespaces properly
 //
 // cf. https://lore.kernel.org/git/pMV5dJabxOBTD8kJBaPuWK0aS6OJhRQ7YFGwfhPCeSJEbPDrIFBza36nXBCgUCeUJWGmpjPI1rlOGvZJEh71Ruz4SqljndUwOCoBUDRHRDU=@eagain.st/
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace(caps: &client::Capabilities) -> bool {
     static MIN_GIT_VERSION_NAMESPACES: Lazy<Version> =
-        Lazy::new(|| Version::new("2.31.0").unwrap());
+        Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version < *MIN_GIT_VERSION_NAMESPACES)
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v6 2/2] fix post_receive to send updated refs in request_pull

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220830045938.93431-3-keepsimple@gmail.com>
In-Reply-To: <20220830045938.93431-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +11 -8

---
 cli/gitd-lib/src/hooks.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/cli/gitd-lib/src/hooks.rs b/cli/gitd-lib/src/hooks.rs
index 5e928ffd..cf37d71f 100644
--- a/cli/gitd-lib/src/hooks.rs
+++ b/cli/gitd-lib/src/hooks.rs
@@ -77,6 +77,16 @@ where
         E: std::error::Error + Send + 'static,
         P: ProgressReporter<Error = E>,
     {
+        // update the signed refs before a possible request_pull,
+        // so that the peer can receive the latest refs.
+        let at = update_signed_refs(
+            reporter,
+            self.spawner.clone(),
+            self.pool.clone(),
+            urn.clone(),
+        )
+        .await?;
+
         if self.post_receive.request_pull {
             tracing::info!("executing request-pull");
             request_pull(reporter, &self.client, &self.seeds, urn.clone()).await?;
@@ -87,14 +97,7 @@ where
             )
             .await?;
         }
-        let at = match update_signed_refs(
-            reporter,
-            self.spawner.clone(),
-            self.pool.clone(),
-            urn.clone(),
-        )
-        .await?
-        {
+        let at = match at {
             Some(at) => at,
             None => return Ok(()),
         };
-- 
2.32.0 (Apple Git-132)

Re: [PATCH radicle-link v5 0/5] Add integration test for bins and fix bugs found

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqezvfEkig_y2wpFB8GBF-dN8ANvBuE6cSy8=7d1VYKEUQ@mail.gmail.com>
In-Reply-To: <CMIR1M1UGAZY.11OMO37UUIDXM@haptop> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Mon, Aug 29, 2022 at 12:13 PM Fintan Halpenny
<fintan.halpenny@gmail.com> wrote:
>
> I have some stylistic suggestions for test file below.
>
> I think you can also squash the changes to that same file into a
> single commit. So, the patch should look like the addition of the
> integration_test.rs file and the fix to the signed_refs update.

Thanks Fintan for your review. I've updated the patch to adopt your
stylistic suggestions. And also squashed all changes into two commits
like you said. Let me know if you have any questions. The 6th version
is sent.

Han

>
> ---
>  bins/tests/integration_test.rs | 165 ++++++++++++++++++++-------------
>  1 file changed, 100 insertions(+), 65 deletions(-)
>
> diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
> index d245748d..91d4d423 100644
> --- a/bins/tests/integration_test.rs
> +++ b/bins/tests/integration_test.rs
> @@ -34,59 +34,79 @@ fn two_peers_and_a_seed() {
>      let seed_home = "/tmp/seed-home";
>      let passphrase = b"play\n";
>
> -    println!("\n== create lnk homes for two peers and one seed ==\n");
> -    let cmd = LnkCmd::ProfileCreate;
> -    run_lnk!(&cmd, peer1_home, passphrase);
> -    run_lnk!(&cmd, peer2_home, passphrase);
> -    run_lnk!(&cmd, seed_home, passphrase);
> +    {
> +        println!("\n== create lnk homes for two peers and one seed ==\n");
> +        let cmd = LnkCmd::ProfileCreate;
> +        run_lnk!(&cmd, peer1_home, passphrase);
> +        run_lnk!(&cmd, peer2_home, passphrase);
> +        run_lnk!(&cmd, seed_home, passphrase);
> +    }
>
> -    println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
> -    let cmd = LnkCmd::ProfileSshAdd;
> -    run_lnk!(&cmd, peer1_home, passphrase);
> -    run_lnk!(&cmd, peer2_home, passphrase);
> -    run_lnk!(&cmd, seed_home, passphrase);
> -
> -    println!("\n== Creating local link 1 identity ==\n");
> -    let peer1_name = "sockpuppet1".to_string();
> -    let cmd = LnkCmd::IdPersonCreate(peer1_name);
> -    let output = run_lnk!(&cmd, peer1_home, passphrase);
> -    let v: Value = serde_json::from_str(&output).unwrap();
> -    let urn1 = v["urn"].as_str().unwrap().to_string();
> -    let cmd = LnkCmd::IdLocalSet(urn1);
> -    run_lnk!(&cmd, peer1_home, passphrase);
> -
> -    println!("\n== Creating local link 2 identity ==\n");
> -    let peer2_name = "sockpuppet2".to_string();
> -    let cmd = LnkCmd::IdPersonCreate(peer2_name);
> -    let output = run_lnk!(&cmd, peer2_home, passphrase);
> -    let v: Value = serde_json::from_str(&output).unwrap();
> -    let urn2 = v["urn"].as_str().unwrap().to_string();
> -    let cmd = LnkCmd::IdLocalSet(urn2);
> -    run_lnk!(&cmd, peer2_home, passphrase);
> +    {
> +        println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
> +        let cmd = LnkCmd::ProfileSshAdd;
> +        run_lnk!(&cmd, peer1_home, passphrase);
> +        run_lnk!(&cmd, peer2_home, passphrase);
> +        run_lnk!(&cmd, seed_home, passphrase);
> +    }
>
> -    println!("\n== Create a local repository ==\n");
> -    let peer1_proj = format!("peer1_proj_{}", timestamp());
> -    let cmd = LnkCmd::IdProjectCreate(peer1_proj.clone());
> -    let output = run_lnk!(&cmd, peer1_home, passphrase);
> -    let v: Value = serde_json::from_str(&output).unwrap();
> -    let proj_urn = v["urn"].as_str().unwrap().to_string();
> -    println!("our project URN: {}", &proj_urn);
> +    {
> +        println!("\n== Creating local link 1 identity ==\n");
> +        let name = "sockpuppet1".to_string();
> +        let cmd = LnkCmd::IdPersonCreate { name };
> +        let output = run_lnk!(&cmd, peer1_home, passphrase);
> +        let v: Value = serde_json::from_str(&output).unwrap();
> +        let urn = v["urn"].as_str().unwrap().to_string();
> +        let cmd = LnkCmd::IdLocalSet { urn: urn };
> +        run_lnk!(&cmd, peer1_home, passphrase);
> +    }
>
> -    println!("\n== Add the seed to the local peer seed configs ==\n");
> -    let cmd = LnkCmd::ProfilePeer;
> -    let seed_peer_id = run_lnk!(&cmd, seed_home, passphrase);
> -    let seed_endpoint = format!("{}@127.0.0.1:8799", &seed_peer_id);
> +    {
> +        println!("\n== Creating local link 2 identity ==\n");
> +        let name = "sockpuppet2".to_string();
> +        let cmd = LnkCmd::IdPersonCreate { name };
> +        let output = run_lnk!(&cmd, peer2_home, passphrase);
> +        let v: Value = serde_json::from_str(&output).unwrap();
> +        let urn = v["urn"].as_str().unwrap().to_string();
> +        let cmd = LnkCmd::IdLocalSet { urn };
> +        run_lnk!(&cmd, peer2_home, passphrase);
> +    }
> +
> +    let urn = {
> +        println!("\n== Create a local repository ==\n");
> +        let peer1_proj = format!("peer1_proj_{}", timestamp());
> +        let cmd = LnkCmd::IdProjectCreate {
> +            name: peer1_proj.clone(),
> +        };
> +        let output = run_lnk!(&cmd, peer1_home, passphrase);
> +        let v: Value = serde_json::from_str(&output).unwrap();
> +        let proj_urn = v["urn"].as_str().unwrap().to_string();
> +        println!("our project URN: {}", &proj_urn);
> +        proj_urn
> +    };
>
> -    let cmd = LnkCmd::ProfileGet;
> -    let peer1_profile = run_lnk!(&cmd, peer1_home, passphrase);
> -    let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
> -    let mut peer1_f = File::create(peer1_seed).unwrap();
> -    peer1_f.write_all(seed_endpoint.as_bytes()).unwrap();
> +    let seed_endpoint = {
> +        println!("\n== Add the seed to the local peer seed configs ==\n");
> +        let cmd = LnkCmd::ProfilePeer;
> +        let seed_peer_id = run_lnk!(&cmd, seed_home, passphrase);
> +        format!("{}@127.0.0.1:8799", &seed_peer_id)
> +    };
>
> -    let peer2_profile = run_lnk!(&cmd, peer2_home, passphrase);
> -    let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
> -    let mut peer2_f = File::create(peer2_seed).unwrap();
> -    peer2_f.write_all(seed_endpoint.as_bytes()).unwrap();
> +    {
> +        let cmd = LnkCmd::ProfileGet;
> +        let peer1_profile = run_lnk!(&cmd, peer1_home, passphrase);
> +        let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
> +        let mut peer1_f = File::create(peer1_seed).unwrap();
> +        peer1_f.write_all(seed_endpoint.as_bytes()).unwrap();
> +    }
> +
> +    {
> +        let cmd = LnkCmd::ProfileGet;
> +        let peer2_profile = run_lnk!(&cmd, peer2_home, passphrase);
> +        let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
> +        let mut peer2_f = File::create(peer2_seed).unwrap();
> +        peer2_f.write_all(seed_endpoint.as_bytes()).unwrap();
> +    }
>
>      println!("\n== Start the seed linkd ==\n");
>      let manifest_path = manifest_path();
> @@ -99,7 +119,7 @@ fn two_peers_and_a_seed() {
>      let mut gitd = spawn_lnk_gitd(peer1_home, &manifest_path, gitd_addr);
>
>      println!("\n== Make some changes in the repo ==\n");
> -    env::set_current_dir(&peer1_proj).unwrap();
> +    env::set_current_dir(&urn).unwrap();
>      let mut test_file = File::create("test").unwrap();
>      test_file.write_all(b"test").unwrap();
>      Command::new("git")
> @@ -116,7 +136,7 @@ fn two_peers_and_a_seed() {
>      println!("git-commit: {:?}", &output);
>
>      println!("\n== Add the linkd remote to the repo ==\n");
> -    let remote_url = format!("ssh://rad@{}/{}.git", gitd_addr, &proj_urn);
> +    let remote_url = format!("ssh://rad@{}/{}.git", gitd_addr, &urn);
>      Command::new("git")
>          .arg("remote")
>          .arg("add")
> @@ -138,11 +158,12 @@ fn two_peers_and_a_seed() {
>
>      env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
>      let peer2_proj = format!("peer2_proj_{}", timestamp());
> -    let cmd = LnkCmd::Clone(proj_urn, peer1_peer_id, peer2_proj.clone());
> +    let cmd = LnkCmd::Clone {
> +        urn,
> +        peer_id: peer1_peer_id,
> +        path: peer2_proj.clone(),
> +    };
>      run_lnk!(&cmd, peer2_home, passphrase);
> -    if !is_parent {
> -        return;
> -    }
>
>      env::set_current_dir(peer2_proj).unwrap();
>      let peer2_last_commit = git_last_commit();
> @@ -162,10 +183,20 @@ enum LnkCmd {
>      ProfileGet,
>      ProfilePeer,
>      ProfileSshAdd,
> -    IdPersonCreate(String),  // the associated string is "the person's name".
> -    IdLocalSet(String),      // the associated string is "urn".
> -    IdProjectCreate(String), // the associated string is "the project name".
> -    Clone(String, String, String), // the associated string is "urn", "peer_id", "path"
> +    IdPersonCreate {
> +        name: String,
> +    },
> +    IdLocalSet {
> +        urn: String,
> +    },
> +    IdProjectCreate {
> +        name: String,
> +    },
> +    Clone {
> +        urn: String,
> +        peer_id: String,
> +        path: String,
> +    },
>  }
>
>  /// Runs a `lnk` command of `$cmd` using `$lnk_home` as the node home.
> @@ -199,12 +230,12 @@ fn process_lnk_output(lnk_home: &str, lnk_process: &mut Master, cmd: &LnkCmd) ->
>          // Print the output and decode them if necessary.
>          println!("{}: {}", lnk_home, line);
>          match cmd {
> -            LnkCmd::IdPersonCreate(ref _name) => {
> +            LnkCmd::IdPersonCreate { .. } => {
>                  if line.find("\"urn\":").is_some() {
>                      output = line; // get the line with URN.
>                  }
>              },
> -            LnkCmd::IdProjectCreate(ref _name) => {
> +            LnkCmd::IdProjectCreate { .. } => {
>                  if line.find("\"urn\":").is_some() {
>                      output = line; // get the line with URN.
>                  }
> @@ -215,7 +246,7 @@ fn process_lnk_output(lnk_home: &str, lnk_process: &mut Master, cmd: &LnkCmd) ->
>              LnkCmd::ProfilePeer => {
>                  output = line; // get the last line for peer id.
>              },
> -            LnkCmd::Clone(ref _urn, ref _peer, ref _path) => {
> +            LnkCmd::Clone { .. } => {
>                  output = line;
>              },
>              _ => {},
> @@ -245,7 +276,7 @@ fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
>          LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
>          LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
>          LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
> -        LnkCmd::IdPersonCreate(name) => {
> +        LnkCmd::IdPersonCreate { name } => {
>              let payload = json!({ "name": name });
>              lnk_cmd
>                  .arg("identities")
> @@ -255,13 +286,13 @@ fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
>                  .arg("--payload")
>                  .arg(payload.to_string())
>          },
> -        LnkCmd::IdLocalSet(urn) => lnk_cmd
> +        LnkCmd::IdLocalSet { urn } => lnk_cmd
>              .arg("identities")
>              .arg("local")
>              .arg("set")
>              .arg("--urn")
>              .arg(urn),
> -        LnkCmd::IdProjectCreate(name) => {
> +        LnkCmd::IdProjectCreate { name } => {
>              let payload = json!({"name": name, "default_branch": "master"});
>              let project_path = format!("./{}", name);
>              lnk_cmd
> @@ -274,7 +305,11 @@ fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
>                  .arg("--payload")
>                  .arg(payload.to_string())
>          },
> -        LnkCmd::Clone(urn, peer_id, peer2_proj) => lnk_cmd
> +        LnkCmd::Clone {
> +            urn,
> +            peer_id,
> +            path: peer2_proj,
> +        } => lnk_cmd
>              .arg("clone")
>              .arg("--urn")
>              .arg(urn)
> --
> 2.31.1

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CMJ8FX95ZGXO.2921YAMFSHUDM@haptop>
In-Reply-To: <20220830045938.93431-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Could you publish the v6 tag, please? :)

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqeviZALmiW4GZxUXJ_7X6bTfL9xurdQxUFaGLSBo=CGgQ@mail.gmail.com>
In-Reply-To: <CMJ8FX95ZGXO.2921YAMFSHUDM@haptop> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Here it is: https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv6

Sorry I forgot it.

Han

On Tue, Aug 30, 2022 at 1:50 AM Fintan Halpenny
<fintan.halpenny@gmail.com> wrote:
>
> Could you publish the v6 tag, please? :)

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CMJIDAXOICOV.3JEMUGB72DNSZ@haptop>
In-Reply-To: <CAEjGaqeviZALmiW4GZxUXJ_7X6bTfL9xurdQxUFaGLSBo=CGgQ@mail.gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Tue Aug 30, 2022 at 1:56 PM IST, Han wrote:
> Here it is: https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv6
>
> Sorry I forgot it.
>
> Han

No worries, thanks for sending it on :)

There's some unrelated (I believe) tests failing when I apply this
patch on top of the latest master. I'm not sure why this is happening
so hesitant to merge it just yet. Could you rebase off origin/master
and see if you see the same behaviour running `scripts/ci/run`?

>
> On Tue, Aug 30, 2022 at 1:50 AM Fintan Halpenny
> <fintan.halpenny@gmail.com> wrote:
> >
> > Could you publish the v6 tag, please? :)

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqcGvWAmd+EkuBvTTsvrz97_7g7r8=D7JjP8+tg3GhVuxQ@mail.gmail.com>
In-Reply-To: <CMJIDAXOICOV.3JEMUGB72DNSZ@haptop> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Tue, Aug 30, 2022 at 9:38 AM Fintan Halpenny
<fintan.halpenny@gmail.com> wrote:
>
> On Tue Aug 30, 2022 at 1:56 PM IST, Han wrote:
> > Here it is: https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv6
> >
> > Sorry I forgot it.
> >
> > Han
>
> No worries, thanks for sending it on :)
>
> There's some unrelated (I believe) tests failing when I apply this
> patch on top of the latest master. I'm not sure why this is happening
> so hesitant to merge it just yet. Could you rebase off origin/master
> and see if you see the same behaviour running `scripts/ci/run`?

I rebased to the latest master and then ran `scripts/ci/run`. Got a
clippy error from the nightly: (It has to use nightly)

error: unnecessary closure used with `bool::then`
  --> git-ref-format/core/src/lit.rs:24:9
   |
24 |         (c.as_ref() == Self::NAME).then(|| Self::SELF)
   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^-------------------
   |                                    |
   |                                    help: use `then_some(..)`
instead: `then_some(Self::SELF)`   |
   = note: `-D clippy::unnecessary-lazy-evaluations` implied by `-D warnings`
   = help: for further information visit
https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_lazy_evaluations

Should I fix this error together with my patch? The script has not
reached to test yet.

>
> >
> > On Tue, Aug 30, 2022 at 1:50 AM Fintan Halpenny
> > <fintan.halpenny@gmail.com> wrote:
> > >
> > > Could you publish the v6 tag, please? :)
>

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqddsjo1Gd8erGNjrJ2VXmxmLxkvpqb5aQK5a=BvngG2-w@mail.gmail.com>
In-Reply-To: <CAEjGaqcGvWAmd+EkuBvTTsvrz97_7g7r8=D7JjP8+tg3GhVuxQ@mail.gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Wed, Aug 31, 2022 at 9:46 AM Han <keepsimple@gmail.com> wrote:
>
> On Tue, Aug 30, 2022 at 9:38 AM Fintan Halpenny
> <fintan.halpenny@gmail.com> wrote:
> >
> > On Tue Aug 30, 2022 at 1:56 PM IST, Han wrote:
> > > Here it is: https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv6
> > >
> > > Sorry I forgot it.
> > >
> > > Han
> >
> > No worries, thanks for sending it on :)
> >
> > There's some unrelated (I believe) tests failing when I apply this
> > patch on top of the latest master. I'm not sure why this is happening
> > so hesitant to merge it just yet. Could you rebase off origin/master
> > and see if you see the same behaviour running `scripts/ci/run`?
>
> I rebased to the latest master and then ran `scripts/ci/run`. Got a
> clippy error from the nightly: (It has to use nightly)
>
> error: unnecessary closure used with `bool::then`
>   --> git-ref-format/core/src/lit.rs:24:9
>    |
> 24 |         (c.as_ref() == Self::NAME).then(|| Self::SELF)
>    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^-------------------
>    |                                    |
>    |                                    help: use `then_some(..)`
> instead: `then_some(Self::SELF)`   |
>    = note: `-D clippy::unnecessary-lazy-evaluations` implied by `-D warnings`
>    = help: for further information visit
> https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_lazy_evaluations
>
> Should I fix this error together with my patch? The script has not
> reached to test yet.
>

I've fixed this one and all other clippy errors locally, and continued
to run `scripts/ci/run`, but eventually the linker failed :

error: linking with `cc` failed: exit status: 1
<snip>
  = note: collect2: fatal error: ld terminated with signal 9 [Killed]
  compilation terminated.

It could be out-of-memory but not 100% sure.  In any case, I cannot
reproduce the test error locally due to this problem, but I will
submit a new patch after the rebase. If you see the error again,
please share a pointer to the error so I can take a look.

thanks
Han


> >
> > >
> > > On Tue, Aug 30, 2022 at 1:50 AM Fintan Halpenny
> > > <fintan.halpenny@gmail.com> wrote:
> > > >
> > > > Could you publish the v6 tag, please? :)
> >

[PATCH radicle-link v7 0/2] Add integration test for bins and fix bugs found

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220831174913.2755-1-keepsimple@gmail.com>
In-Reply-To: <20220830045938.93431-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

This is the 7th version (re-rolling) patch to add an integration test for `bins` package.
It has been rebased on master branch 622c1bcd59a6ce584f957ffe6b874b2af0b207fd.

The test case scenario follows the steps in https://github.com/alexjg/linkd-playground ,
except it is in Rust now and we can just run `cargo test`. To see a more helpful output,
please run `cargo test -- --nocapture` from the `bins/tests` directory.

We also found two bugs along the way: 

1. Git version checks are off. In our testing, it shows that
git 2.25.1 on Ubuntu 20.04 and git 2.30.2 on Debian 11 do not need the workaround that
adds "namespaces" manually. Hence fixing the git version checked against.

2. `request_pull` to the seed fails to pull the latest commit. The reason being that
the signed_refs for `heads/master` is not the latest. A fix is provided.

Han Xu (2):
  Add integration test for bins and fix git version checks
  fix post_receive to send updated refs in request_pull

 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 456 +++++++++++++++++++++++++++++++++
 cli/gitd-lib/src/hooks.rs      |  19 +-
 link-git/src/protocol/fetch.rs |   5 +-
 link-git/src/protocol/ls.rs    |   5 +-
 7 files changed, 518 insertions(+), 10 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v7 1/2] Add integration test for bins and fix git version checks

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220831174913.2755-2-keepsimple@gmail.com>
In-Reply-To: <20220831174913.2755-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +507 -2

---
 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 456 +++++++++++++++++++++++++++++++++
 link-git/src/protocol/fetch.rs |   5 +-
 link-git/src/protocol/ls.rs    |   5 +-
 6 files changed, 507 insertions(+), 2 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

diff --git a/bins/Cargo.lock b/bins/Cargo.lock
index ca8d4229..25d0898b 100644
--- a/bins/Cargo.lock
@@ -840,6 +840,17 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "errno"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e2b2decb0484e15560df3210cf0d78654bb0864b2c138977c07e377a1bae0e2"
+dependencies = [
+ "kernel32-sys",
+ "libc",
+ "winapi 0.2.8",
+]
+
 [[package]]
 name = "event-listener"
 version = "2.5.3"
@@ -2943,6 +2954,16 @@ dependencies = [
  "human_format",
 ]
 
+[[package]]
+name = "pty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50f3d255966981eb4e4c5df3e983e6f7d163221f547406d83b6a460ff5c5ee8"
+dependencies = [
+ "errno",
+ "libc",
+]
+
 [[package]]
 name = "quanta"
 version = "0.4.1"
@@ -3568,6 +3589,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "tests"
+version = "0.1.0"
+dependencies = [
+ "pty",
+ "serde_json",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
diff --git a/bins/Cargo.toml b/bins/Cargo.toml
index 3d1786f4..99b3c69f 100644
--- a/bins/Cargo.toml
@@ -5,4 +5,5 @@ members = [
   "lnk-gitd",
   "lnk-identities-dev",
   "lnk-profile-dev",
+  "tests",
 ]
diff --git a/bins/tests/Cargo.toml b/bins/tests/Cargo.toml
new file mode 100644
index 00000000..3ae82769
--- /dev/null
@@ -0,0 +1,13 @@
+[package]
+name = "tests"
+version = "0.1.0"
+authors = ["Han Xu <keepsimple@gmail.com>"]
+edition = "2021"
+
+[dev-dependencies]
+pty = "0.2"
+serde_json = "1.0"
+
+[[test]]
+name = "integration_test"
+path = "integration_test.rs"
diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
new file mode 100644
index 00000000..2d2a4a92
--- /dev/null
@@ -0,0 +1,456 @@
+// Copyright © 2022 The Radicle Link Contributors
+//
+// This file is part of radicle-link, distributed under the GPLv3 with Radicle
+// Linking Exception. For full terms see the included LICENSE file.
+
+//! Integration test to exercise the programs in `bins`.
+
+use pty::fork::*;
+use serde_json::{json, Value};
+use std::{
+    env,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    process::{Child, Command, Stdio},
+    thread,
+    time::{Duration, SystemTime},
+};
+
+/// This test is inspired by https://github.com/alexjg/linkd-playground
+///
+/// Tests a typical scenario: there are two peer nodes and one seed node.
+/// The main steps are:
+///   - Setup a profile for each node in tmp home directories.
+///   - Setup SSH keys for each profile.
+///   - Create identities.
+///   - Create a local repo for peer 1.
+///   - Start `linkd` for the seed, and `lnk-gitd` for peer 1.
+///   - Push peer 1 repo to its monorepo and to the seed.
+///   - Clone the peer 1 repo to peer 2 via seed.
+///
+/// This test only works in Linux at this moment.
+#[test]
+fn two_peers_and_a_seed() {
+    let peer1_home = "/tmp/link-local-1";
+    let peer2_home = "/tmp/link-local-2";
+    let seed_home = "/tmp/seed-home";
+    let passphrase = b"play\n";
+
+    {
+        println!("\n== create lnk homes for two peers and one seed ==\n");
+        let cmd = LnkCmd::ProfileCreate;
+        run_lnk!(&cmd, peer1_home, passphrase);
+        run_lnk!(&cmd, peer2_home, passphrase);
+        run_lnk!(&cmd, seed_home, passphrase);
+    }
+
+    {
+        println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
+        let cmd = LnkCmd::ProfileSshAdd;
+        run_lnk!(&cmd, peer1_home, passphrase);
+        run_lnk!(&cmd, peer2_home, passphrase);
+        run_lnk!(&cmd, seed_home, passphrase);
+    }
+
+    {
+        println!("\n== Creating local link 1 identity ==\n");
+        let peer1_name = "sockpuppet1".to_string();
+        let cmd = LnkCmd::IdPersonCreate { name: peer1_name };
+        let output = run_lnk!(&cmd, peer1_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let urn = v["urn"].as_str().unwrap().to_string();
+        let cmd = LnkCmd::IdLocalSet { urn };
+        run_lnk!(&cmd, peer1_home, passphrase);
+    }
+
+    {
+        println!("\n== Creating local link 2 identity ==\n");
+        let peer2_name = "sockpuppet2".to_string();
+        let cmd = LnkCmd::IdPersonCreate { name: peer2_name };
+        let output = run_lnk!(&cmd, peer2_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let urn = v["urn"].as_str().unwrap().to_string();
+        let cmd = LnkCmd::IdLocalSet { urn };
+        run_lnk!(&cmd, peer2_home, passphrase);
+    }
+
+    let peer1_proj_dir = format!("peer1_proj_{}", timestamp());
+    let peer1_proj_urn = {
+        println!("\n== Create a local repository for peer1 ==\n");
+        let cmd = LnkCmd::IdProjectCreate {
+            name: peer1_proj_dir.clone(),
+        };
+        let output = run_lnk!(&cmd, peer1_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let proj_urn = v["urn"].as_str().unwrap().to_string();
+        println!("our project URN: {}", &proj_urn);
+        proj_urn
+    };
+
+    println!("\n== Add the seed to the local peer seed configs ==\n");
+    let seed_endpoint = {
+        let cmd = LnkCmd::ProfilePeer;
+        let seed_peer_id = run_lnk!(&cmd, seed_home, passphrase);
+        format!("{}@127.0.0.1:8799", &seed_peer_id)
+    };
+
+    {
+        // Create seed file for peer1
+        let cmd = LnkCmd::ProfileGet;
+        let peer1_profile = run_lnk!(&cmd, peer1_home, passphrase);
+        let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
+        let mut peer1_seed_f = File::create(peer1_seed).unwrap();
+        peer1_seed_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    }
+
+    {
+        // Create seed file for peer2
+        let cmd = LnkCmd::ProfileGet;
+        let peer2_profile = run_lnk!(&cmd, peer2_home, passphrase);
+        let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
+        let mut peer2_seed_f = File::create(peer2_seed).unwrap();
+        peer2_seed_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    }
+
+    println!("\n== Start the seed linkd ==\n");
+    let manifest_path = manifest_path();
+    let mut linkd = spawn_linkd(seed_home, &manifest_path);
+
+    println!("\n== Start the peer 1 gitd ==\n");
+    let cmd = LnkCmd::ProfilePeer;
+    let gitd_addr = "127.0.0.1:9987";
+    let peer1_peer_id = run_lnk!(&cmd, peer1_home, passphrase);
+    let mut gitd = spawn_lnk_gitd(peer1_home, &manifest_path, gitd_addr);
+
+    println!("\n== Make some changes in the repo: add and commit a test file ==\n");
+    env::set_current_dir(&peer1_proj_dir).unwrap();
+    let mut test_file = File::create("test").unwrap();
+    test_file.write_all(b"test").unwrap();
+    Command::new("git")
+        .arg("add")
+        .arg("test")
+        .output()
+        .expect("failed to do git add");
+    let output = Command::new("git")
+        .arg("commit")
+        .arg("-m")
+        .arg("test commit")
+        .output()
+        .expect("failed to do git commit");
+    println!("git-commit: {:?}", &output);
+
+    println!("\n== Add the linkd remote to the repo ==\n");
+    let remote_url = format!("ssh://rad@{}/{}.git", gitd_addr, &peer1_proj_urn);
+    Command::new("git")
+        .arg("remote")
+        .arg("add")
+        .arg("linkd")
+        .arg(remote_url)
+        .output()
+        .expect("failed to do git remote add");
+
+    clean_up_known_hosts();
+
+    if run_git_push_in_child_process() {
+        // The child process is done with git push.
+        return;
+    }
+
+    let peer1_last_commit = git_last_commit();
+
+    println!("\n== Clone to peer2 ==\n");
+
+    env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
+    let peer2_proj = format!("peer2_proj_{}", timestamp());
+    let cmd = LnkCmd::Clone {
+        urn: peer1_proj_urn,
+        peer_id: peer1_peer_id,
+        path: peer2_proj.clone(),
+    };
+    run_lnk!(&cmd, peer2_home, passphrase);
+
+    env::set_current_dir(peer2_proj).unwrap();
+    let peer2_last_commit = git_last_commit();
+    println!("\n== peer1 proj last commit: {}", &peer1_last_commit);
+    println!("\n== peer2 proj last commit: {}", &peer2_last_commit);
+
+    println!("\n== Cleanup: kill linkd (seed) and gitd (peer1) ==\n");
+
+    linkd.kill().ok();
+    gitd.kill().ok();
+
+    assert_eq!(peer1_last_commit, peer2_last_commit);
+}
+
+enum LnkCmd {
+    ProfileCreate,
+    ProfileGet,
+    ProfilePeer,
+    ProfileSshAdd,
+    IdPersonCreate {
+        name: String,
+    },
+    IdLocalSet {
+        urn: String,
+    },
+    IdProjectCreate {
+        name: String,
+    },
+    Clone {
+        urn: String,
+        peer_id: String,
+        path: String,
+    },
+}
+
+/// Runs a `lnk` command of `$cmd` using `$lnk_home` as the node home.
+/// Also support the passphrase input for commands that need it.
+#[macro_export]
+macro_rules! run_lnk {
+    ( $cmd:expr, $lnk_home:ident, $passphrase:ident ) => {{
+        let fork = Fork::from_ptmx().unwrap();
+        if let Some(mut parent) = fork.is_parent().ok() {
+            match $cmd {
+                LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
+                    // Input the passphrase if necessary.
+                    parent.write_all($passphrase).unwrap();
+                },
+                _ => {},
+            }
+            process_lnk_output($lnk_home, &mut parent, $cmd)
+        } else {
+            start_lnk_cmd($lnk_home, $cmd);
+            return;
+        }
+    }};
+}
+
+fn process_lnk_output(lnk_home: &str, lnk_process: &mut Master, cmd: &LnkCmd) -> String {
+    let buf_reader = BufReader::new(lnk_process);
+    let mut output = String::new();
+    for line in buf_reader.lines() {
+        let line = line.unwrap();
+
+        // Print the output and decode them if necessary.
+        println!("{}: {}", lnk_home, line);
+        match cmd {
+            LnkCmd::IdPersonCreate { name: _ } => {
+                if line.find("\"urn\":").is_some() {
+                    output = line; // get the line with URN.
+                }
+            },
+            LnkCmd::IdProjectCreate { name: _ } => {
+                if line.find("\"urn\":").is_some() {
+                    output = line; // get the line with URN.
+                }
+            },
+            LnkCmd::ProfileGet => {
+                output = line; // get the last line for profile id.
+            },
+            LnkCmd::ProfilePeer => {
+                output = line; // get the last line for peer id.
+            },
+            LnkCmd::Clone {
+                urn: _,
+                peer_id: _,
+                path: _,
+            } => {
+                output = line;
+            },
+            _ => {},
+        }
+    }
+
+    output
+}
+
+fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
+    let manifest_path = manifest_path();
+
+    // cargo run \
+    // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
+    // -p lnk -- "$@"
+    let mut lnk_cmd = Command::new("cargo");
+    lnk_cmd
+        .env("LNK_HOME", lnk_home)
+        .arg("run")
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk")
+        .arg("--");
+    let full_cmd = match cmd {
+        LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
+        LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
+        LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
+        LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
+        LnkCmd::IdPersonCreate { name } => {
+            let payload = json!({ "name": name });
+            lnk_cmd
+                .arg("identities")
+                .arg("person")
+                .arg("create")
+                .arg("new")
+                .arg("--payload")
+                .arg(payload.to_string())
+        },
+        LnkCmd::IdLocalSet { urn } => lnk_cmd
+            .arg("identities")
+            .arg("local")
+            .arg("set")
+            .arg("--urn")
+            .arg(urn),
+        LnkCmd::IdProjectCreate { name } => {
+            let payload = json!({"name": name, "default_branch": "master"});
+            let project_path = format!("./{}", name);
+            lnk_cmd
+                .arg("identities")
+                .arg("project")
+                .arg("create")
+                .arg("new")
+                .arg("--path")
+                .arg(project_path)
+                .arg("--payload")
+                .arg(payload.to_string())
+        },
+        LnkCmd::Clone { urn, peer_id, path } => lnk_cmd
+            .arg("clone")
+            .arg("--urn")
+            .arg(urn)
+            .arg("--path")
+            .arg(path)
+            .arg("--peer")
+            .arg(peer_id),
+    };
+    full_cmd.status().expect("lnk cmd failed:");
+}
+
+fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
+    let log_name = format!("linkd_{}.log", &timestamp());
+    let log_file = File::create(&log_name).unwrap();
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/linkd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("linkd")
+        .output()
+        .expect("cargo build linkd failed");
+
+    let child = Command::new(&exec_path)
+        .env("RUST_BACKTRACE", "1")
+        .arg("--lnk-home")
+        .arg(lnk_home)
+        .arg("--track")
+        .arg("everything")
+        .arg("--protocol-listen")
+        .arg("127.0.0.1:8799")
+        .stdout(Stdio::from(log_file))
+        .spawn()
+        .expect("linkd failed to start");
+    println!("linkd stdout redirected to {}", &log_name);
+    thread::sleep(Duration::from_secs(1));
+    child
+}
+
+fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, addr: &str) -> Child {
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk-gitd")
+        .stdout(Stdio::inherit())
+        .output()
+        .expect("cargo build lnk-gitd failed");
+
+    let child = Command::new(&exec_path)
+        .arg(lnk_home)
+        .arg("--push-seeds")
+        .arg("--fetch-seeds")
+        .arg("-a")
+        .arg(addr)
+        .spawn()
+        .expect("lnk-gitd failed to start");
+    println!("started lnk-gitd");
+    child
+}
+
+/// Returns true if runs in the forked child process for git push.
+/// returns false if runs in the parent process.
+fn run_git_push_in_child_process() -> bool {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        let yes = b"yes\n";
+        let buf_reader = BufReader::new(parent);
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("git-push: {}", line);
+            if line.find("key fingerprint").is_some() {
+                parent.write_all(yes).unwrap();
+            }
+        }
+
+        false // This is not the child process.
+    } else {
+        Command::new("git")
+            .arg("push")
+            .arg("linkd")
+            .status()
+            .expect("failed to do git push");
+        true // This is the child process.
+    }
+}
+
+fn git_last_commit() -> String {
+    let output = Command::new("git")
+        .arg("rev-parse")
+        .arg("HEAD")
+        .output()
+        .expect("failed to run git rev-parse");
+    String::from_utf8_lossy(&output.stdout).to_string()
+}
+
+/// Returns UNIX_TIME in millis.
+fn timestamp() -> u128 {
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap();
+    now.as_millis()
+}
+
+/// Returns the full path of `bins` manifest file.
+fn manifest_path() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/Cargo.toml", package_dir.strip_suffix("/tests").unwrap())
+}
+
+/// Returns the full path of `bins/target`.
+fn bins_target_dir() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/target", package_dir.strip_suffix("/tests").unwrap())
+}
+
+fn clean_up_known_hosts() {
+    // ssh-keygen -f "/home/pi/.ssh/known_hosts" -R "[127.0.0.1]:9987"
+    let home_dir = env!("HOME");
+    let known_hosts = format!("{}/.ssh/known_hosts", &home_dir);
+    let output = Command::new("ssh-keygen")
+        .arg("-f")
+        .arg(known_hosts)
+        .arg("-R")
+        .arg("[127.0.0.1]:9987")
+        .output()
+        .expect("failed to do ssh-keygen");
+    println!("ssh-keygen: {:?}", &output);
+}
diff --git a/link-git/src/protocol/fetch.rs b/link-git/src/protocol/fetch.rs
index a5a73065..98b92b51 100644
--- a/link-git/src/protocol/fetch.rs
+++ b/link-git/src/protocol/fetch.rs
@@ -38,8 +38,11 @@ use super::{packwriter::PackWriter, remote_git_version, transport};
 //
 // cf. https://lore.kernel.org/git/CD2XNXHACAXS.13J6JTWZPO1JA@schmidt/
 // Fixed in `git.git` 1ab13eb, which should land in 2.34
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace_want_ref(caps: &client::Capabilities) -> bool {
-    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.33.0").unwrap());
+    static FIXED_AFTER: Lazy<Version> = Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version <= *FIXED_AFTER)
diff --git a/link-git/src/protocol/ls.rs b/link-git/src/protocol/ls.rs
index b3516454..de95b4a3 100644
--- a/link-git/src/protocol/ls.rs
+++ b/link-git/src/protocol/ls.rs
@@ -22,9 +22,12 @@ use super::{remote_git_version, transport};
 // Work around `git-upload-pack` not handling namespaces properly
 //
 // cf. https://lore.kernel.org/git/pMV5dJabxOBTD8kJBaPuWK0aS6OJhRQ7YFGwfhPCeSJEbPDrIFBza36nXBCgUCeUJWGmpjPI1rlOGvZJEh71Ruz4SqljndUwOCoBUDRHRDU=@eagain.st/
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace(caps: &client::Capabilities) -> bool {
     static MIN_GIT_VERSION_NAMESPACES: Lazy<Version> =
-        Lazy::new(|| Version::new("2.31.0").unwrap());
+        Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version < *MIN_GIT_VERSION_NAMESPACES)
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v7 2/2] fix post_receive to send updated refs in request_pull

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220831174913.2755-3-keepsimple@gmail.com>
In-Reply-To: <20220831174913.2755-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +11 -8

---
 cli/gitd-lib/src/hooks.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/cli/gitd-lib/src/hooks.rs b/cli/gitd-lib/src/hooks.rs
index 5e928ffd..cf37d71f 100644
--- a/cli/gitd-lib/src/hooks.rs
+++ b/cli/gitd-lib/src/hooks.rs
@@ -77,6 +77,16 @@ where
         E: std::error::Error + Send + 'static,
         P: ProgressReporter<Error = E>,
     {
+        // update the signed refs before a possible request_pull,
+        // so that the peer can receive the latest refs.
+        let at = update_signed_refs(
+            reporter,
+            self.spawner.clone(),
+            self.pool.clone(),
+            urn.clone(),
+        )
+        .await?;
+
         if self.post_receive.request_pull {
             tracing::info!("executing request-pull");
             request_pull(reporter, &self.client, &self.seeds, urn.clone()).await?;
@@ -87,14 +97,7 @@ where
             )
             .await?;
         }
-        let at = match update_signed_refs(
-            reporter,
-            self.spawner.clone(),
-            self.pool.clone(),
-            urn.clone(),
-        )
-        .await?
-        {
+        let at = match at {
             Some(at) => at,
             None => return Ok(()),
         };
-- 
2.32.0 (Apple Git-132)

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CMKYNKJ1SW9L.2SLXZGHR70WCN@haptop>
In-Reply-To: <CAEjGaqddsjo1Gd8erGNjrJ2VXmxmLxkvpqb5aQK5a=BvngG2-w@mail.gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Wed Aug 31, 2022 at 6:32 PM IST, Han wrote:
> On Wed, Aug 31, 2022 at 9:46 AM Han <keepsimple@gmail.com> wrote:
> >
> > On Tue, Aug 30, 2022 at 9:38 AM Fintan Halpenny
> > <fintan.halpenny@gmail.com> wrote:
> > >
> > > On Tue Aug 30, 2022 at 1:56 PM IST, Han wrote:
> > > > Here it is: https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv6
> > > >
> > > > Sorry I forgot it.
> > > >
> > > > Han
> > >
> > > No worries, thanks for sending it on :)
> > >
> > > There's some unrelated (I believe) tests failing when I apply this
> > > patch on top of the latest master. I'm not sure why this is happening
> > > so hesitant to merge it just yet. Could you rebase off origin/master
> > > and see if you see the same behaviour running `scripts/ci/run`?
> >
> > I rebased to the latest master and then ran `scripts/ci/run`. Got a
> > clippy error from the nightly: (It has to use nightly)
> >
> > error: unnecessary closure used with `bool::then`
> >   --> git-ref-format/core/src/lit.rs:24:9
> >    |
> > 24 |         (c.as_ref() == Self::NAME).then(|| Self::SELF)
> >    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^-------------------
> >    |                                    |
> >    |                                    help: use `then_some(..)`
> > instead: `then_some(Self::SELF)`   |
> >    = note: `-D clippy::unnecessary-lazy-evaluations` implied by `-D warnings`
> >    = help: for further information visit
> > https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_lazy_evaluations
> >
> > Should I fix this error together with my patch? The script has not
> > reached to test yet.
> >
>
> I've fixed this one and all other clippy errors locally, and continued
> to run `scripts/ci/run`, but eventually the linker failed :
>
> error: linking with `cc` failed: exit status: 1
> <snip>
>   = note: collect2: fatal error: ld terminated with signal 9 [Killed]
>   compilation terminated.
>
> It could be out-of-memory but not 100% sure.  In any case, I cannot
> reproduce the test error locally due to this problem, but I will
> submit a new patch after the rebase. If you see the error again,
> please share a pointer to the error so I can take a look.
>
> thanks
> Han


Sorry, I should have been more specific! That's not what I was
referring to. FYI, we use stable for everything apart from formatting
because we use some of the nightly features in the formatting toml
file.

I'll check out your latest changes and try to reproduce the error.

Re: [PATCH radicle-link v7 0/2] Add integration test for bins and fix bugs found

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CMKYPXRDQY0S.3E92KFJBEIZRH@haptop>
In-Reply-To: <20220831174913.2755-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Could you publish your tag? :D

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CMKZYM4S5WIM.EPJ3S4OIO0KV@haptop>
In-Reply-To: <CMKYNKJ1SW9L.2SLXZGHR70WCN@haptop> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Thu Sep 1, 2022 at 10:34 AM IST, Fintan Halpenny wrote:
> On Wed Aug 31, 2022 at 6:32 PM IST, Han wrote:
> > On Wed, Aug 31, 2022 at 9:46 AM Han <keepsimple@gmail.com> wrote:
> > >
> > > On Tue, Aug 30, 2022 at 9:38 AM Fintan Halpenny
> > > <fintan.halpenny@gmail.com> wrote:
> > > >
> > > > On Tue Aug 30, 2022 at 1:56 PM IST, Han wrote:
> > > > > Here it is: https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv6
> > > > >
> > > > > Sorry I forgot it.
> > > > >
> > > > > Han
> > > >
> > > > No worries, thanks for sending it on :)
> > > >
> > > > There's some unrelated (I believe) tests failing when I apply this
> > > > patch on top of the latest master. I'm not sure why this is happening
> > > > so hesitant to merge it just yet. Could you rebase off origin/master
> > > > and see if you see the same behaviour running `scripts/ci/run`?
> > >
> > > I rebased to the latest master and then ran `scripts/ci/run`. Got a
> > > clippy error from the nightly: (It has to use nightly)
> > >
> > > error: unnecessary closure used with `bool::then`
> > >   --> git-ref-format/core/src/lit.rs:24:9
> > >    |
> > > 24 |         (c.as_ref() == Self::NAME).then(|| Self::SELF)
> > >    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^-------------------
> > >    |                                    |
> > >    |                                    help: use `then_some(..)`
> > > instead: `then_some(Self::SELF)`   |
> > >    = note: `-D clippy::unnecessary-lazy-evaluations` implied by `-D warnings`
> > >    = help: for further information visit
> > > https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_lazy_evaluations
> > >
> > > Should I fix this error together with my patch? The script has not
> > > reached to test yet.
> > >
> >
> > I've fixed this one and all other clippy errors locally, and continued
> > to run `scripts/ci/run`, but eventually the linker failed :
> >
> > error: linking with `cc` failed: exit status: 1
> > <snip>
> >   = note: collect2: fatal error: ld terminated with signal 9 [Killed]
> >   compilation terminated.
> >
> > It could be out-of-memory but not 100% sure.  In any case, I cannot
> > reproduce the test error locally due to this problem, but I will
> > submit a new patch after the rebase. If you see the error again,
> > please share a pointer to the error so I can take a look.
> >
> > thanks
> > Han
>
>
> Sorry, I should have been more specific! That's not what I was
> referring to. FYI, we use stable for everything apart from formatting
> because we use some of the nightly features in the formatting toml
> file.
>
> I'll check out your latest changes and try to reproduce the error.

So I can reproduce using your v6 patch by making the following change
to `link-git/t/Cargo.toml`:

     [dev-dependencies.link-git]
     path = ".."
   + features = ["git2"]

and running:

    cargo nextest run --status-level all --failure-output immediate-final --no-fail-fast -p link-git-test

It'll result in a lot of tests failing in the `link-git` test suite.
The *interesting* thing is that I'm also seeing this happening while
porting code over to `radicle-git`[0].

[0]: https://github.com/radicle-dev/radicle-git/pull/1

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CML0WBFETLYY.2KP4K96WOY6A7@haptop>
In-Reply-To: <CMKZYM4S5WIM.EPJ3S4OIO0KV@haptop> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Thu Sep 1, 2022 at 11:36 AM IST, Fintan Halpenny wrote:
> On Thu Sep 1, 2022 at 10:34 AM IST, Fintan Halpenny wrote:
> > On Wed Aug 31, 2022 at 6:32 PM IST, Han wrote:
> > > On Wed, Aug 31, 2022 at 9:46 AM Han <keepsimple@gmail.com> wrote:
> > > >
> > > > On Tue, Aug 30, 2022 at 9:38 AM Fintan Halpenny
> > > > <fintan.halpenny@gmail.com> wrote:
> > > > >
> > > > > On Tue Aug 30, 2022 at 1:56 PM IST, Han wrote:
> > > > > > Here it is: https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv6
> > > > > >
> > > > > > Sorry I forgot it.
> > > > > >
> > > > > > Han
> > > > >
> > > > > No worries, thanks for sending it on :)
> > > > >
> > > > > There's some unrelated (I believe) tests failing when I apply this
> > > > > patch on top of the latest master. I'm not sure why this is happening
> > > > > so hesitant to merge it just yet. Could you rebase off origin/master
> > > > > and see if you see the same behaviour running `scripts/ci/run`?
> > > >
> > > > I rebased to the latest master and then ran `scripts/ci/run`. Got a
> > > > clippy error from the nightly: (It has to use nightly)
> > > >
> > > > error: unnecessary closure used with `bool::then`
> > > >   --> git-ref-format/core/src/lit.rs:24:9
> > > >    |
> > > > 24 |         (c.as_ref() == Self::NAME).then(|| Self::SELF)
> > > >    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^-------------------
> > > >    |                                    |
> > > >    |                                    help: use `then_some(..)`
> > > > instead: `then_some(Self::SELF)`   |
> > > >    = note: `-D clippy::unnecessary-lazy-evaluations` implied by `-D warnings`
> > > >    = help: for further information visit
> > > > https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_lazy_evaluations
> > > >
> > > > Should I fix this error together with my patch? The script has not
> > > > reached to test yet.
> > > >
> > >
> > > I've fixed this one and all other clippy errors locally, and continued
> > > to run `scripts/ci/run`, but eventually the linker failed :
> > >
> > > error: linking with `cc` failed: exit status: 1
> > > <snip>
> > >   = note: collect2: fatal error: ld terminated with signal 9 [Killed]
> > >   compilation terminated.
> > >
> > > It could be out-of-memory but not 100% sure.  In any case, I cannot
> > > reproduce the test error locally due to this problem, but I will
> > > submit a new patch after the rebase. If you see the error again,
> > > please share a pointer to the error so I can take a look.
> > >
> > > thanks
> > > Han
> >
> >
> > Sorry, I should have been more specific! That's not what I was
> > referring to. FYI, we use stable for everything apart from formatting
> > because we use some of the nightly features in the formatting toml
> > file.
> >
> > I'll check out your latest changes and try to reproduce the error.
>
> So I can reproduce using your v6 patch by making the following change
> to `link-git/t/Cargo.toml`:
>
>      [dev-dependencies.link-git]
>      path = ".."
>    + features = ["git2"]
>
> and running:
>
>     cargo nextest run --status-level all --failure-output immediate-final --no-fail-fast -p link-git-test
>
> It'll result in a lot of tests failing in the `link-git` test suite.
> The *interesting* thing is that I'm also seeing this happening while
> porting code over to `radicle-git`[0].
>
> [0]: https://github.com/radicle-dev/radicle-git/pull/1

So I just updated my git installation to 2.36.2 and the tests are
working.  Previously I had `git version 2.31.1`, so I'm wondering if
the change to checking the capabilities wasn't actually correct.

What I find strange though, is that the tests pass on the latest
`master` and it's only under "certain" conditions that they begin to
fail. I'm not sure what those conditions are though. One scenario is
applying your patch and the other is porting the code over to
radicle-git. I'm not sure what those would have in common.

Re: [PATCH radicle-link v7 0/2] Add integration test for bins and fix bugs found

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqdiUMCsm7vrUno50CUz48DAVsMLAKahH8DOtCq38YUo_w@mail.gmail.com>
In-Reply-To: <CMKYPXRDQY0S.3E92KFJBEIZRH@haptop> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Sorry, I always forgot to publish the tag. Here it is:
https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv7


On Thu, Sep 1, 2022 at 2:38 AM Fintan Halpenny
<fintan.halpenny@gmail.com> wrote:
>
> Could you publish your tag? :D

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqeTJrdbYRvd2JR6bwJmZh+hbeF-8sWSVV43q8m6E5LY0A@mail.gmail.com>
In-Reply-To: <CML0WBFETLYY.2KP4K96WOY6A7@haptop> (view parent)
DKIM signature: missing

Download raw message

2 years ago

We have changed the Git version checks in both
link-git/src/protocol/ls.rs and link-git/src/protocol/fetch.rs. I
reverted the fetch.rs change and now link-git tests are passing,
without updating Git in the system. (My system has Git 2.25.1). Our
new Integration Test uses `ls.rs` to do `ls-refs` to obtain refs path,
so that without the fetch.rs change, Integration Test still passes.
Given these observations, I am changing only the `ls.rs` for Git
version check.

I will submit a new patch version, and separate out the Git version
check in its own commit so it is easier to see.

thanks
Han

On Thu, Sep 1, 2022 at 4:23 AM Fintan Halpenny
<fintan.halpenny@gmail.com> wrote:
>
> On Thu Sep 1, 2022 at 11:36 AM IST, Fintan Halpenny wrote:
> > On Thu Sep 1, 2022 at 10:34 AM IST, Fintan Halpenny wrote:
> > > On Wed Aug 31, 2022 at 6:32 PM IST, Han wrote:
> > > > On Wed, Aug 31, 2022 at 9:46 AM Han <keepsimple@gmail.com> wrote:
> > > > >
> > > > > On Tue, Aug 30, 2022 at 9:38 AM Fintan Halpenny
> > > > > <fintan.halpenny@gmail.com> wrote:
> > > > > >
> > > > > > On Tue Aug 30, 2022 at 1:56 PM IST, Han wrote:
> > > > > > > Here it is: https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv6
> > > > > > >
> > > > > > > Sorry I forgot it.
> > > > > > >
> > > > > > > Han
> > > > > >
> > > > > > No worries, thanks for sending it on :)
> > > > > >
> > > > > > There's some unrelated (I believe) tests failing when I apply this
> > > > > > patch on top of the latest master. I'm not sure why this is happening
> > > > > > so hesitant to merge it just yet. Could you rebase off origin/master
> > > > > > and see if you see the same behaviour running `scripts/ci/run`?
> > > > >
> > > > > I rebased to the latest master and then ran `scripts/ci/run`. Got a
> > > > > clippy error from the nightly: (It has to use nightly)
> > > > >
> > > > > error: unnecessary closure used with `bool::then`
> > > > >   --> git-ref-format/core/src/lit.rs:24:9
> > > > >    |
> > > > > 24 |         (c.as_ref() == Self::NAME).then(|| Self::SELF)
> > > > >    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^-------------------
> > > > >    |                                    |
> > > > >    |                                    help: use `then_some(..)`
> > > > > instead: `then_some(Self::SELF)`   |
> > > > >    = note: `-D clippy::unnecessary-lazy-evaluations` implied by `-D warnings`
> > > > >    = help: for further information visit
> > > > > https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_lazy_evaluations
> > > > >
> > > > > Should I fix this error together with my patch? The script has not
> > > > > reached to test yet.
> > > > >
> > > >
> > > > I've fixed this one and all other clippy errors locally, and continued
> > > > to run `scripts/ci/run`, but eventually the linker failed :
> > > >
> > > > error: linking with `cc` failed: exit status: 1
> > > > <snip>
> > > >   = note: collect2: fatal error: ld terminated with signal 9 [Killed]
> > > >   compilation terminated.
> > > >
> > > > It could be out-of-memory but not 100% sure.  In any case, I cannot
> > > > reproduce the test error locally due to this problem, but I will
> > > > submit a new patch after the rebase. If you see the error again,
> > > > please share a pointer to the error so I can take a look.
> > > >
> > > > thanks
> > > > Han
> > >
> > >
> > > Sorry, I should have been more specific! That's not what I was
> > > referring to. FYI, we use stable for everything apart from formatting
> > > because we use some of the nightly features in the formatting toml
> > > file.
> > >
> > > I'll check out your latest changes and try to reproduce the error.
> >
> > So I can reproduce using your v6 patch by making the following change
> > to `link-git/t/Cargo.toml`:
> >
> >      [dev-dependencies.link-git]
> >      path = ".."
> >    + features = ["git2"]
> >
> > and running:
> >
> >     cargo nextest run --status-level all --failure-output immediate-final --no-fail-fast -p link-git-test
> >
> > It'll result in a lot of tests failing in the `link-git` test suite.
> > The *interesting* thing is that I'm also seeing this happening while
> > porting code over to `radicle-git`[0].
> >
> > [0]: https://github.com/radicle-dev/radicle-git/pull/1
>
> So I just updated my git installation to 2.36.2 and the tests are
> working.  Previously I had `git version 2.31.1`, so I'm wondering if
> the change to checking the capabilities wasn't actually correct.
>
> What I find strange though, is that the tests pass on the latest
> `master` and it's only under "certain" conditions that they begin to
> fail. I'm not sure what those conditions are though. One scenario is
> applying your patch and the other is porting the code over to
> radicle-git. I'm not sure what those would have in common.

[PATCH radicle-link v8 0/3] Add integration test for bins and fix bugs found

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220901180749.7976-1-keepsimple@gmail.com>
In-Reply-To: <20220825210251.61675-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

This is the 8th version (re-rolling) patch to add an integration test for `bins` package.
It was refactored the patch into 3 commits so that it is clearer to understand the changes.

The test case scenario follows the steps in https://github.com/alexjg/linkd-playground ,
except it is in Rust now and we can just run `cargo test`. To see a more helpful output,
please run `cargo test -- --nocapture` from the `bins/tests` directory.

We also found two bugs along the way: 

1. Git version checks are off. In our testing, it shows that
git 2.25.1 on Ubuntu 20.04 and git 2.30.2 on Debian 11 do not need the workaround that
adds "namespaces" manually. Hence fixing the git version check in ls-refs.

2. `request_pull` to the seed fails to pull the latest commit. The reason being that
the signed_refs for `heads/master` is not the latest. A fix is provided.


Han Xu (3):
  fix post_receive to send updated refs in request_pull
  Add integration test for bins
  fix Git version check for ls-refs

 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 456 +++++++++++++++++++++++++++++++++
 cli/gitd-lib/src/hooks.rs      |  19 +-
 link-git/src/protocol/ls.rs    |   5 +-
 6 files changed, 514 insertions(+), 9 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v8 1/3] fix post_receive to send updated refs in request_pull

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220901180749.7976-2-keepsimple@gmail.com>
In-Reply-To: <20220901180749.7976-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +11 -8

---
 cli/gitd-lib/src/hooks.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/cli/gitd-lib/src/hooks.rs b/cli/gitd-lib/src/hooks.rs
index 5e928ffd..cf37d71f 100644
--- a/cli/gitd-lib/src/hooks.rs
+++ b/cli/gitd-lib/src/hooks.rs
@@ -77,6 +77,16 @@ where
         E: std::error::Error + Send + 'static,
         P: ProgressReporter<Error = E>,
     {
+        // update the signed refs before a possible request_pull,
+        // so that the peer can receive the latest refs.
+        let at = update_signed_refs(
+            reporter,
+            self.spawner.clone(),
+            self.pool.clone(),
+            urn.clone(),
+        )
+        .await?;
+
         if self.post_receive.request_pull {
             tracing::info!("executing request-pull");
             request_pull(reporter, &self.client, &self.seeds, urn.clone()).await?;
@@ -87,14 +97,7 @@ where
             )
             .await?;
         }
-        let at = match update_signed_refs(
-            reporter,
-            self.spawner.clone(),
-            self.pool.clone(),
-            urn.clone(),
-        )
-        .await?
-        {
+        let at = match at {
             Some(at) => at,
             None => return Ok(()),
         };
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v8 2/3] Add integration test for bins

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220901180749.7976-3-keepsimple@gmail.com>
In-Reply-To: <20220901180749.7976-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +499 -0

---
 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 456 +++++++++++++++++++++++++++++++++
 4 files changed, 499 insertions(+)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

diff --git a/bins/Cargo.lock b/bins/Cargo.lock
index ca8d4229..25d0898b 100644
--- a/bins/Cargo.lock
@@ -840,6 +840,17 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "errno"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e2b2decb0484e15560df3210cf0d78654bb0864b2c138977c07e377a1bae0e2"
+dependencies = [
+ "kernel32-sys",
+ "libc",
+ "winapi 0.2.8",
+]
+
 [[package]]
 name = "event-listener"
 version = "2.5.3"
@@ -2943,6 +2954,16 @@ dependencies = [
  "human_format",
 ]
 
+[[package]]
+name = "pty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50f3d255966981eb4e4c5df3e983e6f7d163221f547406d83b6a460ff5c5ee8"
+dependencies = [
+ "errno",
+ "libc",
+]
+
 [[package]]
 name = "quanta"
 version = "0.4.1"
@@ -3568,6 +3589,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "tests"
+version = "0.1.0"
+dependencies = [
+ "pty",
+ "serde_json",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
diff --git a/bins/Cargo.toml b/bins/Cargo.toml
index 3d1786f4..99b3c69f 100644
--- a/bins/Cargo.toml
@@ -5,4 +5,5 @@ members = [
   "lnk-gitd",
   "lnk-identities-dev",
   "lnk-profile-dev",
+  "tests",
 ]
diff --git a/bins/tests/Cargo.toml b/bins/tests/Cargo.toml
new file mode 100644
index 00000000..3ae82769
--- /dev/null
@@ -0,0 +1,13 @@
+[package]
+name = "tests"
+version = "0.1.0"
+authors = ["Han Xu <keepsimple@gmail.com>"]
+edition = "2021"
+
+[dev-dependencies]
+pty = "0.2"
+serde_json = "1.0"
+
+[[test]]
+name = "integration_test"
+path = "integration_test.rs"
diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
new file mode 100644
index 00000000..2d2a4a92
--- /dev/null
@@ -0,0 +1,456 @@
+// Copyright © 2022 The Radicle Link Contributors
+//
+// This file is part of radicle-link, distributed under the GPLv3 with Radicle
+// Linking Exception. For full terms see the included LICENSE file.
+
+//! Integration test to exercise the programs in `bins`.
+
+use pty::fork::*;
+use serde_json::{json, Value};
+use std::{
+    env,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    process::{Child, Command, Stdio},
+    thread,
+    time::{Duration, SystemTime},
+};
+
+/// This test is inspired by https://github.com/alexjg/linkd-playground
+///
+/// Tests a typical scenario: there are two peer nodes and one seed node.
+/// The main steps are:
+///   - Setup a profile for each node in tmp home directories.
+///   - Setup SSH keys for each profile.
+///   - Create identities.
+///   - Create a local repo for peer 1.
+///   - Start `linkd` for the seed, and `lnk-gitd` for peer 1.
+///   - Push peer 1 repo to its monorepo and to the seed.
+///   - Clone the peer 1 repo to peer 2 via seed.
+///
+/// This test only works in Linux at this moment.
+#[test]
+fn two_peers_and_a_seed() {
+    let peer1_home = "/tmp/link-local-1";
+    let peer2_home = "/tmp/link-local-2";
+    let seed_home = "/tmp/seed-home";
+    let passphrase = b"play\n";
+
+    {
+        println!("\n== create lnk homes for two peers and one seed ==\n");
+        let cmd = LnkCmd::ProfileCreate;
+        run_lnk!(&cmd, peer1_home, passphrase);
+        run_lnk!(&cmd, peer2_home, passphrase);
+        run_lnk!(&cmd, seed_home, passphrase);
+    }
+
+    {
+        println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
+        let cmd = LnkCmd::ProfileSshAdd;
+        run_lnk!(&cmd, peer1_home, passphrase);
+        run_lnk!(&cmd, peer2_home, passphrase);
+        run_lnk!(&cmd, seed_home, passphrase);
+    }
+
+    {
+        println!("\n== Creating local link 1 identity ==\n");
+        let peer1_name = "sockpuppet1".to_string();
+        let cmd = LnkCmd::IdPersonCreate { name: peer1_name };
+        let output = run_lnk!(&cmd, peer1_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let urn = v["urn"].as_str().unwrap().to_string();
+        let cmd = LnkCmd::IdLocalSet { urn };
+        run_lnk!(&cmd, peer1_home, passphrase);
+    }
+
+    {
+        println!("\n== Creating local link 2 identity ==\n");
+        let peer2_name = "sockpuppet2".to_string();
+        let cmd = LnkCmd::IdPersonCreate { name: peer2_name };
+        let output = run_lnk!(&cmd, peer2_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let urn = v["urn"].as_str().unwrap().to_string();
+        let cmd = LnkCmd::IdLocalSet { urn };
+        run_lnk!(&cmd, peer2_home, passphrase);
+    }
+
+    let peer1_proj_dir = format!("peer1_proj_{}", timestamp());
+    let peer1_proj_urn = {
+        println!("\n== Create a local repository for peer1 ==\n");
+        let cmd = LnkCmd::IdProjectCreate {
+            name: peer1_proj_dir.clone(),
+        };
+        let output = run_lnk!(&cmd, peer1_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let proj_urn = v["urn"].as_str().unwrap().to_string();
+        println!("our project URN: {}", &proj_urn);
+        proj_urn
+    };
+
+    println!("\n== Add the seed to the local peer seed configs ==\n");
+    let seed_endpoint = {
+        let cmd = LnkCmd::ProfilePeer;
+        let seed_peer_id = run_lnk!(&cmd, seed_home, passphrase);
+        format!("{}@127.0.0.1:8799", &seed_peer_id)
+    };
+
+    {
+        // Create seed file for peer1
+        let cmd = LnkCmd::ProfileGet;
+        let peer1_profile = run_lnk!(&cmd, peer1_home, passphrase);
+        let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
+        let mut peer1_seed_f = File::create(peer1_seed).unwrap();
+        peer1_seed_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    }
+
+    {
+        // Create seed file for peer2
+        let cmd = LnkCmd::ProfileGet;
+        let peer2_profile = run_lnk!(&cmd, peer2_home, passphrase);
+        let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
+        let mut peer2_seed_f = File::create(peer2_seed).unwrap();
+        peer2_seed_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    }
+
+    println!("\n== Start the seed linkd ==\n");
+    let manifest_path = manifest_path();
+    let mut linkd = spawn_linkd(seed_home, &manifest_path);
+
+    println!("\n== Start the peer 1 gitd ==\n");
+    let cmd = LnkCmd::ProfilePeer;
+    let gitd_addr = "127.0.0.1:9987";
+    let peer1_peer_id = run_lnk!(&cmd, peer1_home, passphrase);
+    let mut gitd = spawn_lnk_gitd(peer1_home, &manifest_path, gitd_addr);
+
+    println!("\n== Make some changes in the repo: add and commit a test file ==\n");
+    env::set_current_dir(&peer1_proj_dir).unwrap();
+    let mut test_file = File::create("test").unwrap();
+    test_file.write_all(b"test").unwrap();
+    Command::new("git")
+        .arg("add")
+        .arg("test")
+        .output()
+        .expect("failed to do git add");
+    let output = Command::new("git")
+        .arg("commit")
+        .arg("-m")
+        .arg("test commit")
+        .output()
+        .expect("failed to do git commit");
+    println!("git-commit: {:?}", &output);
+
+    println!("\n== Add the linkd remote to the repo ==\n");
+    let remote_url = format!("ssh://rad@{}/{}.git", gitd_addr, &peer1_proj_urn);
+    Command::new("git")
+        .arg("remote")
+        .arg("add")
+        .arg("linkd")
+        .arg(remote_url)
+        .output()
+        .expect("failed to do git remote add");
+
+    clean_up_known_hosts();
+
+    if run_git_push_in_child_process() {
+        // The child process is done with git push.
+        return;
+    }
+
+    let peer1_last_commit = git_last_commit();
+
+    println!("\n== Clone to peer2 ==\n");
+
+    env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
+    let peer2_proj = format!("peer2_proj_{}", timestamp());
+    let cmd = LnkCmd::Clone {
+        urn: peer1_proj_urn,
+        peer_id: peer1_peer_id,
+        path: peer2_proj.clone(),
+    };
+    run_lnk!(&cmd, peer2_home, passphrase);
+
+    env::set_current_dir(peer2_proj).unwrap();
+    let peer2_last_commit = git_last_commit();
+    println!("\n== peer1 proj last commit: {}", &peer1_last_commit);
+    println!("\n== peer2 proj last commit: {}", &peer2_last_commit);
+
+    println!("\n== Cleanup: kill linkd (seed) and gitd (peer1) ==\n");
+
+    linkd.kill().ok();
+    gitd.kill().ok();
+
+    assert_eq!(peer1_last_commit, peer2_last_commit);
+}
+
+enum LnkCmd {
+    ProfileCreate,
+    ProfileGet,
+    ProfilePeer,
+    ProfileSshAdd,
+    IdPersonCreate {
+        name: String,
+    },
+    IdLocalSet {
+        urn: String,
+    },
+    IdProjectCreate {
+        name: String,
+    },
+    Clone {
+        urn: String,
+        peer_id: String,
+        path: String,
+    },
+}
+
+/// Runs a `lnk` command of `$cmd` using `$lnk_home` as the node home.
+/// Also support the passphrase input for commands that need it.
+#[macro_export]
+macro_rules! run_lnk {
+    ( $cmd:expr, $lnk_home:ident, $passphrase:ident ) => {{
+        let fork = Fork::from_ptmx().unwrap();
+        if let Some(mut parent) = fork.is_parent().ok() {
+            match $cmd {
+                LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
+                    // Input the passphrase if necessary.
+                    parent.write_all($passphrase).unwrap();
+                },
+                _ => {},
+            }
+            process_lnk_output($lnk_home, &mut parent, $cmd)
+        } else {
+            start_lnk_cmd($lnk_home, $cmd);
+            return;
+        }
+    }};
+}
+
+fn process_lnk_output(lnk_home: &str, lnk_process: &mut Master, cmd: &LnkCmd) -> String {
+    let buf_reader = BufReader::new(lnk_process);
+    let mut output = String::new();
+    for line in buf_reader.lines() {
+        let line = line.unwrap();
+
+        // Print the output and decode them if necessary.
+        println!("{}: {}", lnk_home, line);
+        match cmd {
+            LnkCmd::IdPersonCreate { name: _ } => {
+                if line.find("\"urn\":").is_some() {
+                    output = line; // get the line with URN.
+                }
+            },
+            LnkCmd::IdProjectCreate { name: _ } => {
+                if line.find("\"urn\":").is_some() {
+                    output = line; // get the line with URN.
+                }
+            },
+            LnkCmd::ProfileGet => {
+                output = line; // get the last line for profile id.
+            },
+            LnkCmd::ProfilePeer => {
+                output = line; // get the last line for peer id.
+            },
+            LnkCmd::Clone {
+                urn: _,
+                peer_id: _,
+                path: _,
+            } => {
+                output = line;
+            },
+            _ => {},
+        }
+    }
+
+    output
+}
+
+fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
+    let manifest_path = manifest_path();
+
+    // cargo run \
+    // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
+    // -p lnk -- "$@"
+    let mut lnk_cmd = Command::new("cargo");
+    lnk_cmd
+        .env("LNK_HOME", lnk_home)
+        .arg("run")
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk")
+        .arg("--");
+    let full_cmd = match cmd {
+        LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
+        LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
+        LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
+        LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
+        LnkCmd::IdPersonCreate { name } => {
+            let payload = json!({ "name": name });
+            lnk_cmd
+                .arg("identities")
+                .arg("person")
+                .arg("create")
+                .arg("new")
+                .arg("--payload")
+                .arg(payload.to_string())
+        },
+        LnkCmd::IdLocalSet { urn } => lnk_cmd
+            .arg("identities")
+            .arg("local")
+            .arg("set")
+            .arg("--urn")
+            .arg(urn),
+        LnkCmd::IdProjectCreate { name } => {
+            let payload = json!({"name": name, "default_branch": "master"});
+            let project_path = format!("./{}", name);
+            lnk_cmd
+                .arg("identities")
+                .arg("project")
+                .arg("create")
+                .arg("new")
+                .arg("--path")
+                .arg(project_path)
+                .arg("--payload")
+                .arg(payload.to_string())
+        },
+        LnkCmd::Clone { urn, peer_id, path } => lnk_cmd
+            .arg("clone")
+            .arg("--urn")
+            .arg(urn)
+            .arg("--path")
+            .arg(path)
+            .arg("--peer")
+            .arg(peer_id),
+    };
+    full_cmd.status().expect("lnk cmd failed:");
+}
+
+fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
+    let log_name = format!("linkd_{}.log", &timestamp());
+    let log_file = File::create(&log_name).unwrap();
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/linkd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("linkd")
+        .output()
+        .expect("cargo build linkd failed");
+
+    let child = Command::new(&exec_path)
+        .env("RUST_BACKTRACE", "1")
+        .arg("--lnk-home")
+        .arg(lnk_home)
+        .arg("--track")
+        .arg("everything")
+        .arg("--protocol-listen")
+        .arg("127.0.0.1:8799")
+        .stdout(Stdio::from(log_file))
+        .spawn()
+        .expect("linkd failed to start");
+    println!("linkd stdout redirected to {}", &log_name);
+    thread::sleep(Duration::from_secs(1));
+    child
+}
+
+fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, addr: &str) -> Child {
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk-gitd")
+        .stdout(Stdio::inherit())
+        .output()
+        .expect("cargo build lnk-gitd failed");
+
+    let child = Command::new(&exec_path)
+        .arg(lnk_home)
+        .arg("--push-seeds")
+        .arg("--fetch-seeds")
+        .arg("-a")
+        .arg(addr)
+        .spawn()
+        .expect("lnk-gitd failed to start");
+    println!("started lnk-gitd");
+    child
+}
+
+/// Returns true if runs in the forked child process for git push.
+/// returns false if runs in the parent process.
+fn run_git_push_in_child_process() -> bool {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        let yes = b"yes\n";
+        let buf_reader = BufReader::new(parent);
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("git-push: {}", line);
+            if line.find("key fingerprint").is_some() {
+                parent.write_all(yes).unwrap();
+            }
+        }
+
+        false // This is not the child process.
+    } else {
+        Command::new("git")
+            .arg("push")
+            .arg("linkd")
+            .status()
+            .expect("failed to do git push");
+        true // This is the child process.
+    }
+}
+
+fn git_last_commit() -> String {
+    let output = Command::new("git")
+        .arg("rev-parse")
+        .arg("HEAD")
+        .output()
+        .expect("failed to run git rev-parse");
+    String::from_utf8_lossy(&output.stdout).to_string()
+}
+
+/// Returns UNIX_TIME in millis.
+fn timestamp() -> u128 {
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap();
+    now.as_millis()
+}
+
+/// Returns the full path of `bins` manifest file.
+fn manifest_path() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/Cargo.toml", package_dir.strip_suffix("/tests").unwrap())
+}
+
+/// Returns the full path of `bins/target`.
+fn bins_target_dir() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/target", package_dir.strip_suffix("/tests").unwrap())
+}
+
+fn clean_up_known_hosts() {
+    // ssh-keygen -f "/home/pi/.ssh/known_hosts" -R "[127.0.0.1]:9987"
+    let home_dir = env!("HOME");
+    let known_hosts = format!("{}/.ssh/known_hosts", &home_dir);
+    let output = Command::new("ssh-keygen")
+        .arg("-f")
+        .arg(known_hosts)
+        .arg("-R")
+        .arg("[127.0.0.1]:9987")
+        .output()
+        .expect("failed to do ssh-keygen");
+    println!("ssh-keygen: {:?}", &output);
+}
-- 
2.32.0 (Apple Git-132)

[PATCH radicle-link v8 3/3] fix Git version check for ls-refs

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220901180749.7976-4-keepsimple@gmail.com>
In-Reply-To: <20220901180749.7976-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +4 -1

---
 link-git/src/protocol/ls.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/link-git/src/protocol/ls.rs b/link-git/src/protocol/ls.rs
index b3516454..de95b4a3 100644
--- a/link-git/src/protocol/ls.rs
+++ b/link-git/src/protocol/ls.rs
@@ -22,9 +22,12 @@ use super::{remote_git_version, transport};
 // Work around `git-upload-pack` not handling namespaces properly
 //
 // cf. https://lore.kernel.org/git/pMV5dJabxOBTD8kJBaPuWK0aS6OJhRQ7YFGwfhPCeSJEbPDrIFBza36nXBCgUCeUJWGmpjPI1rlOGvZJEh71Ruz4SqljndUwOCoBUDRHRDU=@eagain.st/
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace(caps: &client::Capabilities) -> bool {
     static MIN_GIT_VERSION_NAMESPACES: Lazy<Version> =
-        Lazy::new(|| Version::new("2.31.0").unwrap());
+        Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version < *MIN_GIT_VERSION_NAMESPACES)
-- 
2.32.0 (Apple Git-132)

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqfQsfZRySUXZW4DoJrKWfKC8_p7AimKUayPSV+=J3Erpw@mail.gmail.com>
In-Reply-To: <CAEjGaqeTJrdbYRvd2JR6bwJmZh+hbeF-8sWSVV43q8m6E5LY0A@mail.gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

I have submitted the new (8th) version of the patch, also published
the tag : https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv8

Cheers,
Han

On Thu, Sep 1, 2022 at 9:47 AM Han <keepsimple@gmail.com> wrote:
>
> We have changed the Git version checks in both
> link-git/src/protocol/ls.rs and link-git/src/protocol/fetch.rs. I
> reverted the fetch.rs change and now link-git tests are passing,
> without updating Git in the system. (My system has Git 2.25.1). Our
> new Integration Test uses `ls.rs` to do `ls-refs` to obtain refs path,
> so that without the fetch.rs change, Integration Test still passes.
> Given these observations, I am changing only the `ls.rs` for Git
> version check.
>
> I will submit a new patch version, and separate out the Git version
> check in its own commit so it is easier to see.
>
> thanks
> Han
>
> On Thu, Sep 1, 2022 at 4:23 AM Fintan Halpenny
> <fintan.halpenny@gmail.com> wrote:
> >
> > On Thu Sep 1, 2022 at 11:36 AM IST, Fintan Halpenny wrote:
> > > On Thu Sep 1, 2022 at 10:34 AM IST, Fintan Halpenny wrote:
> > > > On Wed Aug 31, 2022 at 6:32 PM IST, Han wrote:
> > > > > On Wed, Aug 31, 2022 at 9:46 AM Han <keepsimple@gmail.com> wrote:
> > > > > >
> > > > > > On Tue, Aug 30, 2022 at 9:38 AM Fintan Halpenny
> > > > > > <fintan.halpenny@gmail.com> wrote:
> > > > > > >
> > > > > > > On Tue Aug 30, 2022 at 1:56 PM IST, Han wrote:
> > > > > > > > Here it is: https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv6
> > > > > > > >
> > > > > > > > Sorry I forgot it.
> > > > > > > >
> > > > > > > > Han
> > > > > > >
> > > > > > > No worries, thanks for sending it on :)
> > > > > > >
> > > > > > > There's some unrelated (I believe) tests failing when I apply this
> > > > > > > patch on top of the latest master. I'm not sure why this is happening
> > > > > > > so hesitant to merge it just yet. Could you rebase off origin/master
> > > > > > > and see if you see the same behaviour running `scripts/ci/run`?
> > > > > >
> > > > > > I rebased to the latest master and then ran `scripts/ci/run`. Got a
> > > > > > clippy error from the nightly: (It has to use nightly)
> > > > > >
> > > > > > error: unnecessary closure used with `bool::then`
> > > > > >   --> git-ref-format/core/src/lit.rs:24:9
> > > > > >    |
> > > > > > 24 |         (c.as_ref() == Self::NAME).then(|| Self::SELF)
> > > > > >    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^-------------------
> > > > > >    |                                    |
> > > > > >    |                                    help: use `then_some(..)`
> > > > > > instead: `then_some(Self::SELF)`   |
> > > > > >    = note: `-D clippy::unnecessary-lazy-evaluations` implied by `-D warnings`
> > > > > >    = help: for further information visit
> > > > > > https://rust-lang.github.io/rust-clippy/master/index.html#unnecessary_lazy_evaluations
> > > > > >
> > > > > > Should I fix this error together with my patch? The script has not
> > > > > > reached to test yet.
> > > > > >
> > > > >
> > > > > I've fixed this one and all other clippy errors locally, and continued
> > > > > to run `scripts/ci/run`, but eventually the linker failed :
> > > > >
> > > > > error: linking with `cc` failed: exit status: 1
> > > > > <snip>
> > > > >   = note: collect2: fatal error: ld terminated with signal 9 [Killed]
> > > > >   compilation terminated.
> > > > >
> > > > > It could be out-of-memory but not 100% sure.  In any case, I cannot
> > > > > reproduce the test error locally due to this problem, but I will
> > > > > submit a new patch after the rebase. If you see the error again,
> > > > > please share a pointer to the error so I can take a look.
> > > > >
> > > > > thanks
> > > > > Han
> > > >
> > > >
> > > > Sorry, I should have been more specific! That's not what I was
> > > > referring to. FYI, we use stable for everything apart from formatting
> > > > because we use some of the nightly features in the formatting toml
> > > > file.
> > > >
> > > > I'll check out your latest changes and try to reproduce the error.
> > >
> > > So I can reproduce using your v6 patch by making the following change
> > > to `link-git/t/Cargo.toml`:
> > >
> > >      [dev-dependencies.link-git]
> > >      path = ".."
> > >    + features = ["git2"]
> > >
> > > and running:
> > >
> > >     cargo nextest run --status-level all --failure-output immediate-final --no-fail-fast -p link-git-test
> > >
> > > It'll result in a lot of tests failing in the `link-git` test suite.
> > > The *interesting* thing is that I'm also seeing this happening while
> > > porting code over to `radicle-git`[0].
> > >
> > > [0]: https://github.com/radicle-dev/radicle-git/pull/1
> >
> > So I just updated my git installation to 2.36.2 and the tests are
> > working.  Previously I had `git version 2.31.1`, so I'm wondering if
> > the change to checking the capabilities wasn't actually correct.
> >
> > What I find strange though, is that the tests pass on the latest
> > `master` and it's only under "certain" conditions that they begin to
> > fail. I'm not sure what those conditions are though. One scenario is
> > applying your patch and the other is porting the code over to
> > radicle-git. I'm not sure what those would have in common.

Re: [PATCH radicle-link v8 0/3] Add integration test for bins and fix bugs found

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CMOACSXZORQ0.9XFE8GRUFE6G@haptop>
In-Reply-To: <20220901180749.7976-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Thu Sep 1, 2022 at 7:07 PM IST, Han Xu wrote:
> This is the 8th version (re-rolling) patch to add an integration test for `bins` package.
> It was refactored the patch into 3 commits so that it is clearer to understand the changes.
>
> The test case scenario follows the steps in https://github.com/alexjg/linkd-playground ,
> except it is in Rust now and we can just run `cargo test`. To see a more helpful output,
> please run `cargo test -- --nocapture` from the `bins/tests` directory.
>
> We also found two bugs along the way: 
>
> 1. Git version checks are off. In our testing, it shows that
> git 2.25.1 on Ubuntu 20.04 and git 2.30.2 on Debian 11 do not need the workaround that
> adds "namespaces" manually. Hence fixing the git version check in ls-refs.

Wait, if Debian 11 needs git 2.30.2 then why would we lower the bounds
to 2.25.1? If a Debian 11 computer was running this code and had
2.25.1, then it would say it doesn't need to explicitly namespace but
it actually *does* until 2.30.2. So shouldn't the lower bound be
2.30.1 to maximise the usable OS's? Correct me if I'm wrong :)

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CMOB0P34Q25R.168311ZTVFL2J@haptop>
In-Reply-To: <CAEjGaqfQsfZRySUXZW4DoJrKWfKC8_p7AimKUayPSV+=J3Erpw@mail.gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Also, I noticed that your commits are missing your sign-off, ie. name
and email, as per our DCO policy[0].

[0]: https://github.com/radicle-dev/radicle-link/blob/master/CONTRIBUTING.md#certificate-of-origin

Re: [PATCH radicle-link v8 0/3] Add integration test for bins and fix bugs found

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqfkvtBO0sgxNtcNdSNw3Xjei9E4axk9NC4fUc0nb2pang@mail.gmail.com>
In-Reply-To: <CMOACSXZORQ0.9XFE8GRUFE6G@haptop> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Mon, Sep 5, 2022 at 12:33 AM Fintan Halpenny
<fintan.halpenny@gmail.com> wrote:
>
> On Thu Sep 1, 2022 at 7:07 PM IST, Han Xu wrote:
> > This is the 8th version (re-rolling) patch to add an integration test for `bins` package.
> > It was refactored the patch into 3 commits so that it is clearer to understand the changes.
> >
> > The test case scenario follows the steps in https://github.com/alexjg/linkd-playground ,
> > except it is in Rust now and we can just run `cargo test`. To see a more helpful output,
> > please run `cargo test -- --nocapture` from the `bins/tests` directory.
> >
> > We also found two bugs along the way:
> >
> > 1. Git version checks are off. In our testing, it shows that
> > git 2.25.1 on Ubuntu 20.04 and git 2.30.2 on Debian 11 do not need the workaround that
> > adds "namespaces" manually. Hence fixing the git version check in ls-refs.
>
> Wait, if Debian 11 needs git 2.30.2 then why would we lower the bounds
> to 2.25.1? If a Debian 11 computer was running this code and had
> 2.25.1, then it would say it doesn't need to explicitly namespace but
> it actually *does* until 2.30.2. So shouldn't the lower bound be
> 2.30.1 to maximise the usable OS's? Correct me if I'm wrong :)

I did not mean that Debian 11 _needs_ git 2.30.2 to not use the
workaround.  I meant that Ubuntu 20.04 comes with git 2.25.1 and works
without the workaround for ls-refs. Similarly,  Debian 11 comes with
git 2.30.2 and does not need the workaround.  I believe git 2.25.1
will also work on Debian for ls-defs, even though I didn't try to
lower the git version on Debian to test that. I think Ubuntu 20.04
gives enough confidence.

In other words, I used the lower bound of 2.25.1 so that we can run
the test on Ubuntu 20.04, i.e. maximise the usable OS's.

Re: [PATCH radicle-link v6 0/2] Add integration test for bins and fix bugs found

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqedPqK6nwda5WY3d81rcyWsh9i+XYoefoT7fmfC4S8khg@mail.gmail.com>
In-Reply-To: <CMOB0P34Q25R.168311ZTVFL2J@haptop> (view parent)
DKIM signature: missing

Download raw message

2 years ago

On Mon, Sep 5, 2022 at 12:55 AM Fintan Halpenny
<fintan.halpenny@gmail.com> wrote:
>
> Also, I noticed that your commits are missing your sign-off, ie. name
> and email, as per our DCO policy[0].
>
> [0]: https://github.com/radicle-dev/radicle-link/blob/master/CONTRIBUTING.md#certificate-of-origin

Will do. Thanks for the pointer.

[PATCH radicle-link v9 0/3] Add integration test for bins and fix bugs found

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220906035625.42243-1-keepsimple@gmail.com>
In-Reply-To: <20220825210251.61675-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

This is the 9th version (re-rolling) patch to add an integration test for `bins` package.
The only change from the 8th version is to add sign-off lines in commit messages.

The test case scenario follows the steps in https://github.com/alexjg/linkd-playground ,
except it is in Rust now and we can just run `cargo test`. To see a more helpful output,
please run `cargo test -- --nocapture` from the `bins/tests` directory.

We also found two bugs along the way:

1. Git version checks are off. In our testing, it shows that
git 2.25.1 on Ubuntu 20.04 and git 2.30.2 on Debian 11 do not need the workaround that
adds "namespaces" manually. Hence fixing the git version check in ls-refs.

2. `request_pull` to the seed fails to pull the latest commit. The reason being that
the signed_refs for `heads/master` is not the latest. A fix is provided.

Han Xu (3):
  fix post_receive to send updated refs in request_pull
  Add integration test for bins
  fix Git version check for ls-refs

 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 456 +++++++++++++++++++++++++++++++++
 cli/gitd-lib/src/hooks.rs      |  19 +-
 link-git/src/protocol/ls.rs    |   5 +-
 6 files changed, 514 insertions(+), 9 deletions(-)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

-- 
2.32.1 (Apple Git-133)

[PATCH radicle-link v9 1/3] fix post_receive to send updated refs in request_pull

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220906035625.42243-2-keepsimple@gmail.com>
In-Reply-To: <20220906035625.42243-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +11 -8

Signed-off-by: Han Xu <keepsimple@gmail.com>
---
 cli/gitd-lib/src/hooks.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/cli/gitd-lib/src/hooks.rs b/cli/gitd-lib/src/hooks.rs
index 5e928ffd..cf37d71f 100644
--- a/cli/gitd-lib/src/hooks.rs
+++ b/cli/gitd-lib/src/hooks.rs
@@ -77,6 +77,16 @@ where
         E: std::error::Error + Send + 'static,
         P: ProgressReporter<Error = E>,
     {
+        // update the signed refs before a possible request_pull,
+        // so that the peer can receive the latest refs.
+        let at = update_signed_refs(
+            reporter,
+            self.spawner.clone(),
+            self.pool.clone(),
+            urn.clone(),
+        )
+        .await?;
+
         if self.post_receive.request_pull {
             tracing::info!("executing request-pull");
             request_pull(reporter, &self.client, &self.seeds, urn.clone()).await?;
@@ -87,14 +97,7 @@ where
             )
             .await?;
         }
-        let at = match update_signed_refs(
-            reporter,
-            self.spawner.clone(),
-            self.pool.clone(),
-            urn.clone(),
-        )
-        .await?
-        {
+        let at = match at {
             Some(at) => at,
             None => return Ok(()),
         };
-- 
2.32.1 (Apple Git-133)

[PATCH radicle-link v9 2/3] Add integration test for bins

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220906035625.42243-3-keepsimple@gmail.com>
In-Reply-To: <20220906035625.42243-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +499 -0

Signed-off-by: Han Xu <keepsimple@gmail.com>
---
 bins/Cargo.lock                |  29 +++
 bins/Cargo.toml                |   1 +
 bins/tests/Cargo.toml          |  13 +
 bins/tests/integration_test.rs | 456 +++++++++++++++++++++++++++++++++
 4 files changed, 499 insertions(+)
 create mode 100644 bins/tests/Cargo.toml
 create mode 100644 bins/tests/integration_test.rs

diff --git a/bins/Cargo.lock b/bins/Cargo.lock
index ca8d4229..25d0898b 100644
--- a/bins/Cargo.lock
@@ -840,6 +840,17 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "errno"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e2b2decb0484e15560df3210cf0d78654bb0864b2c138977c07e377a1bae0e2"
+dependencies = [
+ "kernel32-sys",
+ "libc",
+ "winapi 0.2.8",
+]
+
 [[package]]
 name = "event-listener"
 version = "2.5.3"
@@ -2943,6 +2954,16 @@ dependencies = [
  "human_format",
 ]
 
+[[package]]
+name = "pty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50f3d255966981eb4e4c5df3e983e6f7d163221f547406d83b6a460ff5c5ee8"
+dependencies = [
+ "errno",
+ "libc",
+]
+
 [[package]]
 name = "quanta"
 version = "0.4.1"
@@ -3568,6 +3589,14 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "tests"
+version = "0.1.0"
+dependencies = [
+ "pty",
+ "serde_json",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
diff --git a/bins/Cargo.toml b/bins/Cargo.toml
index 3d1786f4..99b3c69f 100644
--- a/bins/Cargo.toml
@@ -5,4 +5,5 @@ members = [
   "lnk-gitd",
   "lnk-identities-dev",
   "lnk-profile-dev",
+  "tests",
 ]
diff --git a/bins/tests/Cargo.toml b/bins/tests/Cargo.toml
new file mode 100644
index 00000000..3ae82769
--- /dev/null
@@ -0,0 +1,13 @@
+[package]
+name = "tests"
+version = "0.1.0"
+authors = ["Han Xu <keepsimple@gmail.com>"]
+edition = "2021"
+
+[dev-dependencies]
+pty = "0.2"
+serde_json = "1.0"
+
+[[test]]
+name = "integration_test"
+path = "integration_test.rs"
diff --git a/bins/tests/integration_test.rs b/bins/tests/integration_test.rs
new file mode 100644
index 00000000..2d2a4a92
--- /dev/null
@@ -0,0 +1,456 @@
+// Copyright © 2022 The Radicle Link Contributors
+//
+// This file is part of radicle-link, distributed under the GPLv3 with Radicle
+// Linking Exception. For full terms see the included LICENSE file.
+
+//! Integration test to exercise the programs in `bins`.
+
+use pty::fork::*;
+use serde_json::{json, Value};
+use std::{
+    env,
+    fs::File,
+    io::{BufRead, BufReader, Write},
+    process::{Child, Command, Stdio},
+    thread,
+    time::{Duration, SystemTime},
+};
+
+/// This test is inspired by https://github.com/alexjg/linkd-playground
+///
+/// Tests a typical scenario: there are two peer nodes and one seed node.
+/// The main steps are:
+///   - Setup a profile for each node in tmp home directories.
+///   - Setup SSH keys for each profile.
+///   - Create identities.
+///   - Create a local repo for peer 1.
+///   - Start `linkd` for the seed, and `lnk-gitd` for peer 1.
+///   - Push peer 1 repo to its monorepo and to the seed.
+///   - Clone the peer 1 repo to peer 2 via seed.
+///
+/// This test only works in Linux at this moment.
+#[test]
+fn two_peers_and_a_seed() {
+    let peer1_home = "/tmp/link-local-1";
+    let peer2_home = "/tmp/link-local-2";
+    let seed_home = "/tmp/seed-home";
+    let passphrase = b"play\n";
+
+    {
+        println!("\n== create lnk homes for two peers and one seed ==\n");
+        let cmd = LnkCmd::ProfileCreate;
+        run_lnk!(&cmd, peer1_home, passphrase);
+        run_lnk!(&cmd, peer2_home, passphrase);
+        run_lnk!(&cmd, seed_home, passphrase);
+    }
+
+    {
+        println!("\n== add ssh keys for each profile to the ssh-agent ==\n");
+        let cmd = LnkCmd::ProfileSshAdd;
+        run_lnk!(&cmd, peer1_home, passphrase);
+        run_lnk!(&cmd, peer2_home, passphrase);
+        run_lnk!(&cmd, seed_home, passphrase);
+    }
+
+    {
+        println!("\n== Creating local link 1 identity ==\n");
+        let peer1_name = "sockpuppet1".to_string();
+        let cmd = LnkCmd::IdPersonCreate { name: peer1_name };
+        let output = run_lnk!(&cmd, peer1_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let urn = v["urn"].as_str().unwrap().to_string();
+        let cmd = LnkCmd::IdLocalSet { urn };
+        run_lnk!(&cmd, peer1_home, passphrase);
+    }
+
+    {
+        println!("\n== Creating local link 2 identity ==\n");
+        let peer2_name = "sockpuppet2".to_string();
+        let cmd = LnkCmd::IdPersonCreate { name: peer2_name };
+        let output = run_lnk!(&cmd, peer2_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let urn = v["urn"].as_str().unwrap().to_string();
+        let cmd = LnkCmd::IdLocalSet { urn };
+        run_lnk!(&cmd, peer2_home, passphrase);
+    }
+
+    let peer1_proj_dir = format!("peer1_proj_{}", timestamp());
+    let peer1_proj_urn = {
+        println!("\n== Create a local repository for peer1 ==\n");
+        let cmd = LnkCmd::IdProjectCreate {
+            name: peer1_proj_dir.clone(),
+        };
+        let output = run_lnk!(&cmd, peer1_home, passphrase);
+        let v: Value = serde_json::from_str(&output).unwrap();
+        let proj_urn = v["urn"].as_str().unwrap().to_string();
+        println!("our project URN: {}", &proj_urn);
+        proj_urn
+    };
+
+    println!("\n== Add the seed to the local peer seed configs ==\n");
+    let seed_endpoint = {
+        let cmd = LnkCmd::ProfilePeer;
+        let seed_peer_id = run_lnk!(&cmd, seed_home, passphrase);
+        format!("{}@127.0.0.1:8799", &seed_peer_id)
+    };
+
+    {
+        // Create seed file for peer1
+        let cmd = LnkCmd::ProfileGet;
+        let peer1_profile = run_lnk!(&cmd, peer1_home, passphrase);
+        let peer1_seed = format!("{}/{}/seeds", peer1_home, peer1_profile);
+        let mut peer1_seed_f = File::create(peer1_seed).unwrap();
+        peer1_seed_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    }
+
+    {
+        // Create seed file for peer2
+        let cmd = LnkCmd::ProfileGet;
+        let peer2_profile = run_lnk!(&cmd, peer2_home, passphrase);
+        let peer2_seed = format!("{}/{}/seeds", peer2_home, peer2_profile);
+        let mut peer2_seed_f = File::create(peer2_seed).unwrap();
+        peer2_seed_f.write_all(seed_endpoint.as_bytes()).unwrap();
+    }
+
+    println!("\n== Start the seed linkd ==\n");
+    let manifest_path = manifest_path();
+    let mut linkd = spawn_linkd(seed_home, &manifest_path);
+
+    println!("\n== Start the peer 1 gitd ==\n");
+    let cmd = LnkCmd::ProfilePeer;
+    let gitd_addr = "127.0.0.1:9987";
+    let peer1_peer_id = run_lnk!(&cmd, peer1_home, passphrase);
+    let mut gitd = spawn_lnk_gitd(peer1_home, &manifest_path, gitd_addr);
+
+    println!("\n== Make some changes in the repo: add and commit a test file ==\n");
+    env::set_current_dir(&peer1_proj_dir).unwrap();
+    let mut test_file = File::create("test").unwrap();
+    test_file.write_all(b"test").unwrap();
+    Command::new("git")
+        .arg("add")
+        .arg("test")
+        .output()
+        .expect("failed to do git add");
+    let output = Command::new("git")
+        .arg("commit")
+        .arg("-m")
+        .arg("test commit")
+        .output()
+        .expect("failed to do git commit");
+    println!("git-commit: {:?}", &output);
+
+    println!("\n== Add the linkd remote to the repo ==\n");
+    let remote_url = format!("ssh://rad@{}/{}.git", gitd_addr, &peer1_proj_urn);
+    Command::new("git")
+        .arg("remote")
+        .arg("add")
+        .arg("linkd")
+        .arg(remote_url)
+        .output()
+        .expect("failed to do git remote add");
+
+    clean_up_known_hosts();
+
+    if run_git_push_in_child_process() {
+        // The child process is done with git push.
+        return;
+    }
+
+    let peer1_last_commit = git_last_commit();
+
+    println!("\n== Clone to peer2 ==\n");
+
+    env::set_current_dir("..").unwrap(); // out of the peer1 proj directory.
+    let peer2_proj = format!("peer2_proj_{}", timestamp());
+    let cmd = LnkCmd::Clone {
+        urn: peer1_proj_urn,
+        peer_id: peer1_peer_id,
+        path: peer2_proj.clone(),
+    };
+    run_lnk!(&cmd, peer2_home, passphrase);
+
+    env::set_current_dir(peer2_proj).unwrap();
+    let peer2_last_commit = git_last_commit();
+    println!("\n== peer1 proj last commit: {}", &peer1_last_commit);
+    println!("\n== peer2 proj last commit: {}", &peer2_last_commit);
+
+    println!("\n== Cleanup: kill linkd (seed) and gitd (peer1) ==\n");
+
+    linkd.kill().ok();
+    gitd.kill().ok();
+
+    assert_eq!(peer1_last_commit, peer2_last_commit);
+}
+
+enum LnkCmd {
+    ProfileCreate,
+    ProfileGet,
+    ProfilePeer,
+    ProfileSshAdd,
+    IdPersonCreate {
+        name: String,
+    },
+    IdLocalSet {
+        urn: String,
+    },
+    IdProjectCreate {
+        name: String,
+    },
+    Clone {
+        urn: String,
+        peer_id: String,
+        path: String,
+    },
+}
+
+/// Runs a `lnk` command of `$cmd` using `$lnk_home` as the node home.
+/// Also support the passphrase input for commands that need it.
+#[macro_export]
+macro_rules! run_lnk {
+    ( $cmd:expr, $lnk_home:ident, $passphrase:ident ) => {{
+        let fork = Fork::from_ptmx().unwrap();
+        if let Some(mut parent) = fork.is_parent().ok() {
+            match $cmd {
+                LnkCmd::ProfileCreate | LnkCmd::ProfileSshAdd => {
+                    // Input the passphrase if necessary.
+                    parent.write_all($passphrase).unwrap();
+                },
+                _ => {},
+            }
+            process_lnk_output($lnk_home, &mut parent, $cmd)
+        } else {
+            start_lnk_cmd($lnk_home, $cmd);
+            return;
+        }
+    }};
+}
+
+fn process_lnk_output(lnk_home: &str, lnk_process: &mut Master, cmd: &LnkCmd) -> String {
+    let buf_reader = BufReader::new(lnk_process);
+    let mut output = String::new();
+    for line in buf_reader.lines() {
+        let line = line.unwrap();
+
+        // Print the output and decode them if necessary.
+        println!("{}: {}", lnk_home, line);
+        match cmd {
+            LnkCmd::IdPersonCreate { name: _ } => {
+                if line.find("\"urn\":").is_some() {
+                    output = line; // get the line with URN.
+                }
+            },
+            LnkCmd::IdProjectCreate { name: _ } => {
+                if line.find("\"urn\":").is_some() {
+                    output = line; // get the line with URN.
+                }
+            },
+            LnkCmd::ProfileGet => {
+                output = line; // get the last line for profile id.
+            },
+            LnkCmd::ProfilePeer => {
+                output = line; // get the last line for peer id.
+            },
+            LnkCmd::Clone {
+                urn: _,
+                peer_id: _,
+                path: _,
+            } => {
+                output = line;
+            },
+            _ => {},
+        }
+    }
+
+    output
+}
+
+fn start_lnk_cmd(lnk_home: &str, cmd: &LnkCmd) {
+    let manifest_path = manifest_path();
+
+    // cargo run \
+    // --manifest-path $LINK_CHECKOUT/bins/Cargo.toml \
+    // -p lnk -- "$@"
+    let mut lnk_cmd = Command::new("cargo");
+    lnk_cmd
+        .env("LNK_HOME", lnk_home)
+        .arg("run")
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk")
+        .arg("--");
+    let full_cmd = match cmd {
+        LnkCmd::ProfileCreate => lnk_cmd.arg("profile").arg("create"),
+        LnkCmd::ProfileGet => lnk_cmd.arg("profile").arg("get"),
+        LnkCmd::ProfilePeer => lnk_cmd.arg("profile").arg("peer"),
+        LnkCmd::ProfileSshAdd => lnk_cmd.arg("profile").arg("ssh").arg("add"),
+        LnkCmd::IdPersonCreate { name } => {
+            let payload = json!({ "name": name });
+            lnk_cmd
+                .arg("identities")
+                .arg("person")
+                .arg("create")
+                .arg("new")
+                .arg("--payload")
+                .arg(payload.to_string())
+        },
+        LnkCmd::IdLocalSet { urn } => lnk_cmd
+            .arg("identities")
+            .arg("local")
+            .arg("set")
+            .arg("--urn")
+            .arg(urn),
+        LnkCmd::IdProjectCreate { name } => {
+            let payload = json!({"name": name, "default_branch": "master"});
+            let project_path = format!("./{}", name);
+            lnk_cmd
+                .arg("identities")
+                .arg("project")
+                .arg("create")
+                .arg("new")
+                .arg("--path")
+                .arg(project_path)
+                .arg("--payload")
+                .arg(payload.to_string())
+        },
+        LnkCmd::Clone { urn, peer_id, path } => lnk_cmd
+            .arg("clone")
+            .arg("--urn")
+            .arg(urn)
+            .arg("--path")
+            .arg(path)
+            .arg("--peer")
+            .arg(peer_id),
+    };
+    full_cmd.status().expect("lnk cmd failed:");
+}
+
+fn spawn_linkd(lnk_home: &str, manifest_path: &str) -> Child {
+    let log_name = format!("linkd_{}.log", &timestamp());
+    let log_file = File::create(&log_name).unwrap();
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/linkd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("linkd")
+        .output()
+        .expect("cargo build linkd failed");
+
+    let child = Command::new(&exec_path)
+        .env("RUST_BACKTRACE", "1")
+        .arg("--lnk-home")
+        .arg(lnk_home)
+        .arg("--track")
+        .arg("everything")
+        .arg("--protocol-listen")
+        .arg("127.0.0.1:8799")
+        .stdout(Stdio::from(log_file))
+        .spawn()
+        .expect("linkd failed to start");
+    println!("linkd stdout redirected to {}", &log_name);
+    thread::sleep(Duration::from_secs(1));
+    child
+}
+
+fn spawn_lnk_gitd(lnk_home: &str, manifest_path: &str, addr: &str) -> Child {
+    let target_dir = bins_target_dir();
+    let exec_path = format!("{}/debug/lnk-gitd", &target_dir);
+
+    Command::new("cargo")
+        .arg("build")
+        .arg("--target-dir")
+        .arg(&target_dir)
+        .arg("--manifest-path")
+        .arg(manifest_path)
+        .arg("-p")
+        .arg("lnk-gitd")
+        .stdout(Stdio::inherit())
+        .output()
+        .expect("cargo build lnk-gitd failed");
+
+    let child = Command::new(&exec_path)
+        .arg(lnk_home)
+        .arg("--push-seeds")
+        .arg("--fetch-seeds")
+        .arg("-a")
+        .arg(addr)
+        .spawn()
+        .expect("lnk-gitd failed to start");
+    println!("started lnk-gitd");
+    child
+}
+
+/// Returns true if runs in the forked child process for git push.
+/// returns false if runs in the parent process.
+fn run_git_push_in_child_process() -> bool {
+    let fork = Fork::from_ptmx().unwrap();
+    if let Some(mut parent) = fork.is_parent().ok() {
+        let yes = b"yes\n";
+        let buf_reader = BufReader::new(parent);
+        for line in buf_reader.lines() {
+            let line = line.unwrap();
+            println!("git-push: {}", line);
+            if line.find("key fingerprint").is_some() {
+                parent.write_all(yes).unwrap();
+            }
+        }
+
+        false // This is not the child process.
+    } else {
+        Command::new("git")
+            .arg("push")
+            .arg("linkd")
+            .status()
+            .expect("failed to do git push");
+        true // This is the child process.
+    }
+}
+
+fn git_last_commit() -> String {
+    let output = Command::new("git")
+        .arg("rev-parse")
+        .arg("HEAD")
+        .output()
+        .expect("failed to run git rev-parse");
+    String::from_utf8_lossy(&output.stdout).to_string()
+}
+
+/// Returns UNIX_TIME in millis.
+fn timestamp() -> u128 {
+    let now = SystemTime::now()
+        .duration_since(SystemTime::UNIX_EPOCH)
+        .unwrap();
+    now.as_millis()
+}
+
+/// Returns the full path of `bins` manifest file.
+fn manifest_path() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/Cargo.toml", package_dir.strip_suffix("/tests").unwrap())
+}
+
+/// Returns the full path of `bins/target`.
+fn bins_target_dir() -> String {
+    let package_dir = env!("CARGO_MANIFEST_DIR");
+    format!("{}/target", package_dir.strip_suffix("/tests").unwrap())
+}
+
+fn clean_up_known_hosts() {
+    // ssh-keygen -f "/home/pi/.ssh/known_hosts" -R "[127.0.0.1]:9987"
+    let home_dir = env!("HOME");
+    let known_hosts = format!("{}/.ssh/known_hosts", &home_dir);
+    let output = Command::new("ssh-keygen")
+        .arg("-f")
+        .arg(known_hosts)
+        .arg("-R")
+        .arg("[127.0.0.1]:9987")
+        .output()
+        .expect("failed to do ssh-keygen");
+    println!("ssh-keygen: {:?}", &output);
+}
-- 
2.32.1 (Apple Git-133)

[PATCH radicle-link v9 3/3] fix Git version check for ls-refs

Han Xu <keepsimple@gmail.com>

Details

Message ID: <20220906035625.42243-4-keepsimple@gmail.com>
In-Reply-To: <20220906035625.42243-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

Patch: +4 -1

Signed-off-by: Han Xu <keepsimple@gmail.com>
---
 link-git/src/protocol/ls.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/link-git/src/protocol/ls.rs b/link-git/src/protocol/ls.rs
index b3516454..de95b4a3 100644
--- a/link-git/src/protocol/ls.rs
+++ b/link-git/src/protocol/ls.rs
@@ -22,9 +22,12 @@ use super::{remote_git_version, transport};
 // Work around `git-upload-pack` not handling namespaces properly
 //
 // cf. https://lore.kernel.org/git/pMV5dJabxOBTD8kJBaPuWK0aS6OJhRQ7YFGwfhPCeSJEbPDrIFBza36nXBCgUCeUJWGmpjPI1rlOGvZJEh71Ruz4SqljndUwOCoBUDRHRDU=@eagain.st/
+//
+// Based on testing with git 2.25.1 in Ubuntu 20.04, this workaround is
+// not needed. Hence the checked version is lowered to 2.25.0.
 fn must_namespace(caps: &client::Capabilities) -> bool {
     static MIN_GIT_VERSION_NAMESPACES: Lazy<Version> =
-        Lazy::new(|| Version::new("2.31.0").unwrap());
+        Lazy::new(|| Version::new("2.25.0").unwrap());
 
     remote_git_version(caps)
         .map(|version| version < *MIN_GIT_VERSION_NAMESPACES)
-- 
2.32.1 (Apple Git-133)

Re: [PATCH radicle-link v9 0/3] Add integration test for bins and fix bugs found

Fintan Halpenny <fintan.halpenny@gmail.com>

Details

Message ID: <CMP7JYSPCPA9.ZRWKJ7TL9ZIF@haptop>
In-Reply-To: <20220906035625.42243-1-keepsimple@gmail.com> (view parent)
DKIM signature: missing

Download raw message

2 years ago

LGTM! Can you publish the tag? :)

Re: [PATCH radicle-link v9 0/3] Add integration test for bins and fix bugs found

Han <keepsimple@gmail.com>

Details

Message ID: <CAEjGaqfc5A1LdH-JT_X=azEfN9vb4=vsSubj8o7WyygzLCybrA@mail.gmail.com>
In-Reply-To: <CMP7JYSPCPA9.ZRWKJ7TL9ZIF@haptop> (view parent)
DKIM signature: missing

Download raw message

2 years ago

sorry I never remember to publish a tag ;-) Here it is:

https://github.com/keepsimple1/radicle-link/releases/tag/patches%2Fintegration-test-bins%2Fv9

On Tue, Sep 6, 2022 at 2:24 AM Fintan Halpenny
<fintan.halpenny@gmail.com> wrote:
>
> LGTM! Can you publish the tag? :)