diff --git a/Cargo.toml b/Cargo.toml
index df73a393782fb60769b8eb6d8295ef546fb7b6fb..6e71ae55b967076c58a5ef9da9020af57fe4a4d0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -51,10 +51,6 @@ ark-secq256k1 = "0.4.0"
ark-vesta = "0.4.0"
criterion = "0.3"
-[[bench]]
-name = "recoding"
-harness = false
-
[[example]]
name = "bench_commit"
path = "examples/benches/commit.rs"
@@ -78,3 +74,7 @@ path = "examples/benches/setup.rs"
[[example]]
name = "bench_linalg"
path = "examples/benches/linalg.rs"
+
+[[example]]
+name = "bench_recoding"
+path = "examples/benches/recoding.rs"
diff --git a/README.md b/README.md
index e1d9558b0c0bb861420bdeca128044b61a939ac5..cd4f411ccd8a417b3a65d6a30642d6a91139a682 100644
--- a/README.md
+++ b/README.md
@@ -19,3 +19,6 @@ tests for the binary application can also be run with
```bash
nu tests/cli.nu
```
+
+## the benchmarks
+see [the `README`](examples/benches/README.md)
diff --git a/benches/recoding.rs b/benches/recoding.rs
deleted file mode 100644
index f623f6c17cf564b521ab8cd33d7c24828ede9db1..0000000000000000000000000000000000000000
--- a/benches/recoding.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-// see `benches/README.md`
-use std::time::Duration;
-
-use ark_ff::PrimeField;
-use ark_std::rand::Rng;
-
-use criterion::{criterion_group, criterion_main, Criterion};
-
-use komodo::{
- fec::{recode_with_coeffs, Shard},
- field,
-};
-
-fn to_curve<F: PrimeField>(n: u128) -> F {
- F::from_le_bytes_mod_order(&n.to_le_bytes())
-}
-
-fn create_fake_shard<F: PrimeField>(nb_bytes: usize, k: usize) -> Shard<F> {
- let mut rng = rand::thread_rng();
- let bytes: Vec<u8> = (0..nb_bytes).map(|_| rng.gen::<u8>()).collect();
-
- let linear_combination: Vec<F> = (0..k).map(|_| to_curve::<F>(rng.gen::<u128>())).collect();
-
- Shard {
- k: k as u32,
- linear_combination,
- hash: vec![],
- data: field::split_data_into_field_elements::<F>(&bytes, 1),
- size: 0,
- }
-}
-
-fn bench_template<F: PrimeField>(
- c: &mut Criterion,
- nb_bytes: usize,
- k: usize,
- nb_shards: usize,
- curve: &str,
-) {
- let shards: Vec<Shard<F>> = (0..nb_shards)
- .map(|_| create_fake_shard(nb_bytes, k))
- .collect();
-
- let mut rng = rand::thread_rng();
- let coeffs: Vec<F> = (0..nb_shards)
- .map(|_| to_curve::<F>(rng.gen::<u128>()))
- .collect();
-
- c.bench_function(
- &format!(
- "recoding {} bytes and {} shards with k = {} on {}",
- nb_bytes, nb_shards, k, curve
- ),
- |b| b.iter(|| recode_with_coeffs(&shards, &coeffs)),
- );
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
- for nb_bytes in [1, 1_024, 1_024 * 1_024] {
- for nb_shards in [2, 4, 8, 16] {
- for k in [2, 4, 8, 16] {
- bench_template::<ark_bls12_381::Fr>(c, nb_bytes, k, nb_shards, "BLS-12-381");
- bench_template::<ark_bn254::Fr>(c, nb_bytes, k, nb_shards, "BN-254");
- bench_template::<ark_pallas::Fr>(c, nb_bytes, k, nb_shards, "PALLAS");
- }
- }
- }
-}
-
-criterion_group!(
- name = benches;
- config = Criterion::default()
- .warm_up_time(Duration::from_secs_f32(0.5))
- .sample_size(10);
- targets = criterion_benchmark
-);
-criterion_main!(benches);
diff --git a/benches/README.md b/examples/benches/README.md
similarity index 65%
rename from benches/README.md
rename to examples/benches/README.md
index 4f04dc131270ba8ac2a25bfe067e8c26b92c1a02..b123bab220291c6e782ce83d81622959b87d3c54 100644
--- a/benches/README.md
+++ b/examples/benches/README.md
@@ -1,5 +1,6 @@
```nushell
use scripts/math.nu *
+use scripts/formats.nu *
```
## atomic operations
@@ -62,9 +63,8 @@ for graph in [
(
$linalg
| where op == $graph.op
- | rename --column { n: "x", name: "curve", mean: "measurement", stddev: "error" }
- | group-by curve --to-table
- | update items { reject curve }
+ | rename --column { n: "x", mean: "measurement", stddev: "error" }
+ | group-by name --to-table
| to json
)
]
@@ -86,15 +86,14 @@ python scripts/plot/plot.py ...[
| ns-to-ms $.times
| compute-stats $.times
| insert degree { get label | parse "degree {d}" | into record | get d | into int}
- | insert curve {|it| if ($it.name | str starts-with "ARK") {
+ | update name {|it| if ($it.name | str starts-with "ARK") {
let c = $it.name | parse "ARK setup on {curve}" | into record | get curve
$"($c)-ark"
} else {
$it.name | parse "setup on {curve}" | into record | get curve
}}
| rename --column { degree: "x", mean: "measurement", stddev: "error" }
- | group-by curve --to-table
- | update items { reject curve }
+ | group-by name --to-table
| to json
)
]
@@ -114,11 +113,52 @@ python scripts/plot/plot.py ...[
open commit.ndjson
| ns-to-ms $.times
| compute-stats $.times
- | update label { parse "degree {d}" | into record | get d | into int }
- | rename --column { label: "x", name: "curve", mean: "measurement", stddev: "error" }
- | group-by curve --to-table
- | update items { reject curve }
+ | insert degree { get label | parse "degree {d}" | into record | get d | into int }
+ | rename --column { degree: "x", mean: "measurement", stddev: "error" }
+ | group-by name --to-table
| to json
)
]
```
+
+## end-to-end benchmarks
+### recoding
+```nushell
+cargo run --example bench_recoding -- ...[
+ --nb-measurements 10
+ ...[1, 1_024, (1_024 * 1_024)]
+ --shards ...[2, 4, 8, 16]
+ --ks ...[2, 4, 8, 16]
+] | from ndnuon | to ndjson out> recoding.ndjson
+```
+```nushell
+python scripts/plot/plot.py --title "recoding with k = 4" (
+ open recoding.ndjson
+ | ns-to-ms $.times
+ | compute-stats $.times
+ | update label { from nuon }
+ | flatten --all label
+ | insert case { $"($in.name) / ($in.shards)" }
+ | where k == 4 # $k$ has a negligible influence on _recoding_
+ | rename --column { bytes: "x", mean: "measurement", stddev: "error" }
+ | group-by case --to-table
+ | insert style {|it|
+ let g = $it.group | parse "{c} / {s}" | into record | into int s
+ let c = match $g.c {
+ "BLS-12-381" => "blue"
+ "BN-254" => "orange"
+ "PALLAS" => "green"
+ _ => "gray"
+ }
+ let t = match $g.s {
+ 2 => "dotted"
+ 4 => "dashdot"
+ 8 => "dashed"
+ 16 => "solid"
+ _ => { color: "loosely dotted" }
+ }
+ { color: $c, line: { type: $t } }
+ }
+ | to json
+)
+```
diff --git a/examples/benches/recoding.rs b/examples/benches/recoding.rs
new file mode 100644
index 0000000000000000000000000000000000000000..610330ce41fb2d613b24727c7b2d5e763bc1ff47
--- /dev/null
+++ b/examples/benches/recoding.rs
@@ -0,0 +1,83 @@
+// see `benches/README.md`
+use ark_ff::PrimeField;
+use ark_std::rand::Rng;
+
+use clap::{arg, command, Parser};
+use komodo::{
+ fec::{recode_with_coeffs, Shard},
+ field,
+};
+use plnk::Bencher;
+
+fn to_curve<F: PrimeField>(n: u128) -> F {
+ F::from_le_bytes_mod_order(&n.to_le_bytes())
+}
+
+fn create_fake_shard<F: PrimeField>(nb_bytes: usize, k: usize) -> Shard<F> {
+ let mut rng = rand::thread_rng();
+ let bytes: Vec<u8> = (0..nb_bytes).map(|_| rng.gen::<u8>()).collect();
+
+ let linear_combination: Vec<F> = (0..k).map(|_| to_curve::<F>(rng.gen::<u128>())).collect();
+
+ Shard {
+ k: k as u32,
+ linear_combination,
+ hash: vec![],
+ data: field::split_data_into_field_elements::<F>(&bytes, 1),
+ size: 0,
+ }
+}
+
+fn bench_template<F: PrimeField>(b: &Bencher, nb_bytes: usize, k: usize, nb_shards: usize) {
+ let shards: Vec<Shard<F>> = (0..nb_shards)
+ .map(|_| create_fake_shard(nb_bytes, k))
+ .collect();
+
+ let mut rng = rand::thread_rng();
+ let coeffs: Vec<F> = (0..nb_shards)
+ .map(|_| to_curve::<F>(rng.gen::<u128>()))
+ .collect();
+
+ plnk::bench(
+ b,
+ &format!(
+ r#"{{"bytes": {}, "shards": {}, "k": {}}}"#,
+ nb_bytes, nb_shards, k
+ ),
+ || plnk::timeit(|| recode_with_coeffs(&shards, &coeffs)),
+ );
+}
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+ #[arg(num_args = 1.., value_delimiter = ' ')]
+ bytes: Vec<usize>,
+
+ #[arg(short, long, num_args = 1.., value_delimiter = ' ')]
+ shards: Vec<usize>,
+
+ #[arg(short, long, num_args = 1.., value_delimiter = ' ')]
+ ks: Vec<usize>,
+
+ /// the number of measurements to repeat each case, larger values will reduce the variance of
+ /// the measurements
+ #[arg(short, long)]
+ nb_measurements: usize,
+}
+
+fn main() {
+ let cli = Cli::parse();
+
+ let bencher = plnk::Bencher::new(cli.nb_measurements);
+
+ for b in cli.bytes {
+ for s in &cli.shards {
+ for k in &cli.ks {
+ bench_template::<ark_bls12_381::Fr>(&bencher.with_name("BLS-12-381"), b, *k, *s);
+ bench_template::<ark_bn254::Fr>(&bencher.with_name("BN-254"), b, *k, *s);
+ bench_template::<ark_pallas::Fr>(&bencher.with_name("PALLAS"), b, *k, *s);
+ }
+ }
+ }
+}
diff --git a/scripts/formats.nu b/scripts/formats.nu
new file mode 100644
index 0000000000000000000000000000000000000000..3b07548c071356aff281c62cec830c3e7b6600ef
--- /dev/null
+++ b/scripts/formats.nu
@@ -0,0 +1,15 @@
+export def "from ndnuon" []: [string -> any] {
+ lines | each { from nuon }
+}
+
+export def "to ndnuon" []: [any -> string] {
+ each { to nuon --raw } | to text
+}
+
+export def "from nuonl" []: [string -> any] {
+ from ndnuon
+}
+
+export def "to nuonl" []: [any -> string] {
+ to ndnuon
+}