From aaf86cd1742485034ad85212b8e5940625be9abb Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Wed, 20 Mar 2024 17:08:29 +0100
Subject: [PATCH 01/17] make `Matrix::invert` public
---
src/linalg.rs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/linalg.rs b/src/linalg.rs
index 48cac2a4..558f8913 100644
--- a/src/linalg.rs
+++ b/src/linalg.rs
@@ -135,7 +135,7 @@ impl<T: Field> Matrix<T> {
}
}
- pub(super) fn invert(&self) -> Result<Self, KomodoError> {
+ pub fn invert(&self) -> Result<Self, KomodoError> {
if self.height != self.width {
return Err(KomodoError::NonSquareMatrix(self.height, self.width));
}
--
GitLab
From 2622e2942a1901851a7d36636fae821b29bedfe5 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Wed, 20 Mar 2024 17:04:53 +0100
Subject: [PATCH 02/17] add `linalg` benchmark and criterion
---
Cargo.toml | 5 +++++
benches/linalg.rs | 22 ++++++++++++++++++++++
2 files changed, 27 insertions(+)
create mode 100644 benches/linalg.rs
diff --git a/Cargo.toml b/Cargo.toml
index 6b2b45d0..9e65fa3b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -21,3 +21,8 @@ tracing-subscriber = "0.3.17"
[dev-dependencies]
rand = "0.8.5"
+criterion = "0.3"
+
+[[bench]]
+name = "linalg"
+harness = false
diff --git a/benches/linalg.rs b/benches/linalg.rs
new file mode 100644
index 00000000..fdec2439
--- /dev/null
+++ b/benches/linalg.rs
@@ -0,0 +1,22 @@
+use ark_bls12_381::Bls12_381;
+use ark_ec::pairing::Pairing;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use komodo::linalg::Matrix;
+
+fn bench_template<E: Pairing>(c: &mut Criterion, n: usize) {
+ let matrix = Matrix::<E::ScalarField>::random(n, n);
+
+ c.bench_function(&format!("inverse {}x{}", n, n), |b| {
+ b.iter(|| matrix.invert().unwrap())
+ });
+}
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+ for n in [10, 20, 30, 50, 100, 200] {
+ bench_template::<Bls12_381>(c, black_box(n));
+ }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
--
GitLab
From f5b9beaf99de4e8502b0a7d72be029f52e716d28 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 11:23:16 +0100
Subject: [PATCH 03/17] minor tweaks to the benchmark
---
benches/linalg.rs | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/benches/linalg.rs b/benches/linalg.rs
index fdec2439..79ebbf90 100644
--- a/benches/linalg.rs
+++ b/benches/linalg.rs
@@ -1,10 +1,10 @@
use ark_bls12_381::Bls12_381;
use ark_ec::pairing::Pairing;
-
use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
use komodo::linalg::Matrix;
-fn bench_template<E: Pairing>(c: &mut Criterion, n: usize) {
+fn inverse_template<E: Pairing>(c: &mut Criterion, n: usize) {
let matrix = Matrix::<E::ScalarField>::random(n, n);
c.bench_function(&format!("inverse {}x{}", n, n), |b| {
@@ -12,11 +12,11 @@ fn bench_template<E: Pairing>(c: &mut Criterion, n: usize) {
});
}
-pub fn criterion_benchmark(c: &mut Criterion) {
- for n in [10, 20, 30, 50, 100, 200] {
- bench_template::<Bls12_381>(c, black_box(n));
+fn inverse(c: &mut Criterion) {
+ for n in [10, 20, 40, 80, 160, 320] {
+ inverse_template::<Bls12_381>(c, black_box(n));
}
}
-criterion_group!(benches, criterion_benchmark);
+criterion_group!(benches, inverse);
criterion_main!(benches);
--
GitLab
From ae9b4e146d802ad367c8a3b985a99f31325316db Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 11:26:36 +0100
Subject: [PATCH 04/17] add curve name to benchmark name
---
benches/linalg.rs | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/benches/linalg.rs b/benches/linalg.rs
index 79ebbf90..07634451 100644
--- a/benches/linalg.rs
+++ b/benches/linalg.rs
@@ -7,9 +7,10 @@ use komodo::linalg::Matrix;
fn inverse_template<E: Pairing>(c: &mut Criterion, n: usize) {
let matrix = Matrix::<E::ScalarField>::random(n, n);
- c.bench_function(&format!("inverse {}x{}", n, n), |b| {
- b.iter(|| matrix.invert().unwrap())
- });
+ c.bench_function(
+ &format!("inverse {}x{} on {}", n, n, std::any::type_name::<E>()),
+ |b| b.iter(|| matrix.invert().unwrap()),
+ );
}
fn inverse(c: &mut Criterion) {
--
GitLab
From c71f9ecc43d7a906eed45d00d996eacd3811da8d Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 11:45:40 +0100
Subject: [PATCH 05/17] add benchmark for `Matrix::transpose`
---
benches/linalg.rs | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/benches/linalg.rs b/benches/linalg.rs
index 07634451..3414eb80 100644
--- a/benches/linalg.rs
+++ b/benches/linalg.rs
@@ -19,5 +19,21 @@ fn inverse(c: &mut Criterion) {
}
}
+fn transpose_template<E: Pairing>(c: &mut Criterion, n: usize) {
+ let matrix = Matrix::<E::ScalarField>::random(n, n);
+
+ c.bench_function(
+ &format!("transpose {}x{} on {}", n, n, std::any::type_name::<E>()),
+ |b| b.iter(|| matrix.transpose()),
+ );
+}
+
+fn transpose(c: &mut Criterion) {
+ for n in [10, 20, 40, 80, 160, 320] {
+ transpose_template::<Bls12_381>(c, black_box(n));
+ }
+}
+
criterion_group!(benches, inverse);
+criterion_group!(benches, transpose);
criterion_main!(benches);
--
GitLab
From f302177c1783bb4064f1bae32a0e75e6104e17bb Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 11:46:23 +0100
Subject: [PATCH 06/17] make `Matrix::transpose` public
---
src/linalg.rs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/linalg.rs b/src/linalg.rs
index 558f8913..557a9b5f 100644
--- a/src/linalg.rs
+++ b/src/linalg.rs
@@ -239,7 +239,7 @@ impl<T: Field> Matrix<T> {
})
}
- pub(super) fn transpose(&self) -> Self {
+ pub fn transpose(&self) -> Self {
let height = self.width;
let width = self.height;
--
GitLab
From a75aaf7a942b77572d436ead6282bc7671d0ebb9 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 11:47:12 +0100
Subject: [PATCH 07/17] fix criterion
---
benches/linalg.rs | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/benches/linalg.rs b/benches/linalg.rs
index 3414eb80..66039fd8 100644
--- a/benches/linalg.rs
+++ b/benches/linalg.rs
@@ -34,6 +34,5 @@ fn transpose(c: &mut Criterion) {
}
}
-criterion_group!(benches, inverse);
-criterion_group!(benches, transpose);
+criterion_group!(benches, inverse, transpose);
criterion_main!(benches);
--
GitLab
From bc36582bcf2757a6e4293e5413224c81d3a4793c Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 11:48:52 +0100
Subject: [PATCH 08/17] make `Matrix::mul` public
---
src/linalg.rs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/linalg.rs b/src/linalg.rs
index 557a9b5f..26e726ba 100644
--- a/src/linalg.rs
+++ b/src/linalg.rs
@@ -209,7 +209,7 @@ impl<T: Field> Matrix<T> {
nb_non_zero_rows
}
- pub(super) fn mul(&self, rhs: &Self) -> Result<Self, KomodoError> {
+ pub fn mul(&self, rhs: &Self) -> Result<Self, KomodoError> {
if self.width != rhs.height {
return Err(KomodoError::IncompatibleMatrixShapes(
self.height,
--
GitLab
From fa0df045b2af9976a8bd5ad785988cc084cc8726 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 11:49:45 +0100
Subject: [PATCH 09/17] add mul benchmark
---
benches/linalg.rs | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/benches/linalg.rs b/benches/linalg.rs
index 66039fd8..3e439f73 100644
--- a/benches/linalg.rs
+++ b/benches/linalg.rs
@@ -34,5 +34,21 @@ fn transpose(c: &mut Criterion) {
}
}
-criterion_group!(benches, inverse, transpose);
+fn mul_template<E: Pairing>(c: &mut Criterion, n: usize) {
+ let mat_a = Matrix::<E::ScalarField>::random(n, n);
+ let mat_b = Matrix::<E::ScalarField>::random(n, n);
+
+ c.bench_function(
+ &format!("mul {}x{} on {}", n, n, std::any::type_name::<E>()),
+ |b| b.iter(|| mat_a.mul(&mat_b)),
+ );
+}
+
+fn mul(c: &mut Criterion) {
+ for n in [10, 20, 40, 80, 160, 320] {
+ mul_template::<Bls12_381>(c, black_box(n));
+ }
+}
+
+criterion_group!(benches, inverse, transpose, mul);
criterion_main!(benches);
--
GitLab
From 284734a56076b7130d8aad00a0f6be345b405828 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 12:56:19 +0100
Subject: [PATCH 10/17] add a python script to plot the results
---
benches/plot.py | 43 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 43 insertions(+)
create mode 100644 benches/plot.py
diff --git a/benches/plot.py b/benches/plot.py
new file mode 100644
index 00000000..7eb810f7
--- /dev/null
+++ b/benches/plot.py
@@ -0,0 +1,43 @@
+import matplotlib.pyplot as plt
+import json
+from typing import Any, Dict, List
+
+RESULTS_FILE = "results.ndjson"
+NB_NS_IN_MS = 1e6
+
+
+def plot(data: List[Dict[str, Any]], label: str, ax):
+ filtered_data = list(filter(lambda line: line["id"].startswith(label), data))
+
+ sizes = [
+ int(line["id"].split(' ')[1].split('x')[0]) for line in filtered_data
+ ]
+ means = [
+ line["mean"]["estimate"] / NB_NS_IN_MS for line in filtered_data
+ ]
+
+ ax.plot(sizes, means, label=label)
+
+
+if __name__ == "__main__":
+ with open(RESULTS_FILE, "r") as file:
+ data = list(filter(
+ lambda line: line["reason"] == "benchmark-complete",
+ map(
+ json.loads,
+ file.readlines()
+ )
+ ))
+
+ labels = ["transpose", "mul", "inverse"]
+
+ fig, axs = plt.subplots(len(labels), 1)
+
+ for label, ax in zip(labels, axs):
+ plot(data, label, ax)
+ ax.set_yscale("log")
+ ax.set_ylabel("time in ms")
+ ax.legend()
+ ax.grid()
+
+ plt.show()
--
GitLab
From f9fb5e96de7c39ab9c29a6c4a4fc685d29c7d2e0 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 13:01:53 +0100
Subject: [PATCH 11/17] show the upper and lower bounds
---
benches/plot.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/benches/plot.py b/benches/plot.py
index 7eb810f7..b1268cfd 100644
--- a/benches/plot.py
+++ b/benches/plot.py
@@ -15,8 +15,15 @@ def plot(data: List[Dict[str, Any]], label: str, ax):
means = [
line["mean"]["estimate"] / NB_NS_IN_MS for line in filtered_data
]
+ up = [
+ line["mean"]["upper_bound"] / NB_NS_IN_MS for line in filtered_data
+ ]
+ down = [
+ line["mean"]["lower_bound"] / NB_NS_IN_MS for line in filtered_data
+ ]
ax.plot(sizes, means, label=label)
+ ax.fill_between(sizes, down, up, color="gray", alpha=0.3, label="bounds")
if __name__ == "__main__":
--
GitLab
From 8692cef83e5591e17ac2fe54b1e6060bfd4128ea Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 13:08:58 +0100
Subject: [PATCH 12/17] add median and refactor
---
benches/plot.py | 29 ++++++++++++++++++-----------
1 file changed, 18 insertions(+), 11 deletions(-)
diff --git a/benches/plot.py b/benches/plot.py
index b1268cfd..2a0e880f 100644
--- a/benches/plot.py
+++ b/benches/plot.py
@@ -6,24 +6,30 @@ RESULTS_FILE = "results.ndjson"
NB_NS_IN_MS = 1e6
+def extract(data: List[Dict[str, Any]], k1: str, k2: str) -> List[float]:
+ return [line[k1][k2] / NB_NS_IN_MS for line in data]
+
+
def plot(data: List[Dict[str, Any]], label: str, ax):
filtered_data = list(filter(lambda line: line["id"].startswith(label), data))
sizes = [
int(line["id"].split(' ')[1].split('x')[0]) for line in filtered_data
]
- means = [
- line["mean"]["estimate"] / NB_NS_IN_MS for line in filtered_data
- ]
- up = [
- line["mean"]["upper_bound"] / NB_NS_IN_MS for line in filtered_data
- ]
- down = [
- line["mean"]["lower_bound"] / NB_NS_IN_MS for line in filtered_data
- ]
- ax.plot(sizes, means, label=label)
- ax.fill_between(sizes, down, up, color="gray", alpha=0.3, label="bounds")
+ means = extract(filtered_data, "mean", "estimate")
+ up = extract(filtered_data, "mean", "upper_bound")
+ down = extract(filtered_data, "mean", "lower_bound")
+
+ ax.plot(sizes, means, label="mean", color="blue")
+ ax.fill_between(sizes, down, up, color="blue", alpha=0.3, label="mean bounds")
+
+ medians = extract(filtered_data, "median", "estimate")
+ up = extract(filtered_data, "median", "upper_bound")
+ down = extract(filtered_data, "median", "lower_bound")
+
+ ax.plot(sizes, medians, label="median", color="orange")
+ ax.fill_between(sizes, down, up, color="orange", alpha=0.3, label="median bounds")
if __name__ == "__main__":
@@ -42,6 +48,7 @@ if __name__ == "__main__":
for label, ax in zip(labels, axs):
plot(data, label, ax)
+ ax.set_title(label)
ax.set_yscale("log")
ax.set_ylabel("time in ms")
ax.legend()
--
GitLab
From 7c1ec44cdd850720f0ba561172b25495c229da69 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 13:10:24 +0100
Subject: [PATCH 13/17] refactor
---
benches/plot.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/benches/plot.py b/benches/plot.py
index 2a0e880f..fdc1e989 100644
--- a/benches/plot.py
+++ b/benches/plot.py
@@ -10,8 +10,8 @@ def extract(data: List[Dict[str, Any]], k1: str, k2: str) -> List[float]:
return [line[k1][k2] / NB_NS_IN_MS for line in data]
-def plot(data: List[Dict[str, Any]], label: str, ax):
- filtered_data = list(filter(lambda line: line["id"].startswith(label), data))
+def plot(data: List[Dict[str, Any]], key: str, ax):
+ filtered_data = list(filter(lambda line: line["id"].startswith(key), data))
sizes = [
int(line["id"].split(' ')[1].split('x')[0]) for line in filtered_data
@@ -47,7 +47,7 @@ if __name__ == "__main__":
fig, axs = plt.subplots(len(labels), 1)
for label, ax in zip(labels, axs):
- plot(data, label, ax)
+ plot(data, key=label, ax=ax)
ax.set_title(label)
ax.set_yscale("log")
ax.set_ylabel("time in ms")
--
GitLab
From f7395a5810058b10cbfbbeba436c8ae26005aba6 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 13:16:28 +0100
Subject: [PATCH 14/17] pass filename as argument and refactor
---
benches/plot.py | 32 +++++++++++++++++++++++++++-----
1 file changed, 27 insertions(+), 5 deletions(-)
diff --git a/benches/plot.py b/benches/plot.py
index fdc1e989..08b17c60 100644
--- a/benches/plot.py
+++ b/benches/plot.py
@@ -1,16 +1,19 @@
import matplotlib.pyplot as plt
import json
+import sys
+import os
from typing import Any, Dict, List
-RESULTS_FILE = "results.ndjson"
NB_NS_IN_MS = 1e6
+Data = List[Dict[str, Any]]
-def extract(data: List[Dict[str, Any]], k1: str, k2: str) -> List[float]:
+
+def extract(data: Data, k1: str, k2: str) -> List[float]:
return [line[k1][k2] / NB_NS_IN_MS for line in data]
-def plot(data: List[Dict[str, Any]], key: str, ax):
+def plot(data: Data, key: str, ax):
filtered_data = list(filter(lambda line: line["id"].startswith(key), data))
sizes = [
@@ -32,8 +35,20 @@ def plot(data: List[Dict[str, Any]], key: str, ax):
ax.fill_between(sizes, down, up, color="orange", alpha=0.3, label="median bounds")
-if __name__ == "__main__":
- with open(RESULTS_FILE, "r") as file:
+def parse_args():
+ if len(sys.argv) == 1:
+ print("please give a filename as first positional argument")
+ exit(1)
+
+ return sys.argv[1]
+
+
+def read_data(data_file: str) -> Data:
+ if not os.path.exists(data_file):
+ print(f"no such file: `{data_file}`")
+ exit(1)
+
+ with open(data_file, "r") as file:
data = list(filter(
lambda line: line["reason"] == "benchmark-complete",
map(
@@ -42,6 +57,13 @@ if __name__ == "__main__":
)
))
+ return data
+
+
+if __name__ == "__main__":
+ results_file = parse_args()
+ data = read_data(results_file)
+
labels = ["transpose", "mul", "inverse"]
fig, axs = plt.subplots(len(labels), 1)
--
GitLab
From 58708668fe7b666af32d64e6fbaf8f91d80683c6 Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 13:17:37 +0100
Subject: [PATCH 15/17] add a README
---
benches/README.md | 9 +++++++++
1 file changed, 9 insertions(+)
create mode 100644 benches/README.md
diff --git a/benches/README.md b/benches/README.md
new file mode 100644
index 00000000..9867ccd1
--- /dev/null
+++ b/benches/README.md
@@ -0,0 +1,9 @@
+## run the benchmarks
+```shell
+nushell> cargo criterion --output-format verbose --message-format json out> results.ndjson
+```
+
+## plot the results
+```shell
+python benches/plot.py results.ndjson
+```
--
GitLab
From 707c56833106b7b303241459d658aa159d8e72fc Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 13:17:59 +0100
Subject: [PATCH 16/17] ignore result files
---
.gitignore | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.gitignore b/.gitignore
index 2c96eb1b..e2b32eb7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
target/
Cargo.lock
+
+*.ndjson
--
GitLab
From 32d1655f2a173adf2091d689b77715f0877823ff Mon Sep 17 00:00:00 2001
From: "a.stevan" <antoine.stevan@isae-supaero.fr>
Date: Fri, 22 Mar 2024 13:23:21 +0100
Subject: [PATCH 17/17] add more benchmark points
---
benches/linalg.rs | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/benches/linalg.rs b/benches/linalg.rs
index 3e439f73..21528738 100644
--- a/benches/linalg.rs
+++ b/benches/linalg.rs
@@ -14,7 +14,7 @@ fn inverse_template<E: Pairing>(c: &mut Criterion, n: usize) {
}
fn inverse(c: &mut Criterion) {
- for n in [10, 20, 40, 80, 160, 320] {
+ for n in [10, 15, 20, 30, 40, 60, 80, 120, 160, 240, 320] {
inverse_template::<Bls12_381>(c, black_box(n));
}
}
@@ -29,7 +29,7 @@ fn transpose_template<E: Pairing>(c: &mut Criterion, n: usize) {
}
fn transpose(c: &mut Criterion) {
- for n in [10, 20, 40, 80, 160, 320] {
+ for n in [10, 15, 20, 30, 40, 60, 80, 120, 160, 240, 320] {
transpose_template::<Bls12_381>(c, black_box(n));
}
}
@@ -45,7 +45,7 @@ fn mul_template<E: Pairing>(c: &mut Criterion, n: usize) {
}
fn mul(c: &mut Criterion) {
- for n in [10, 20, 40, 80, 160, 320] {
+ for n in [10, 15, 20, 30, 40, 60, 80, 120, 160, 240, 320] {
mul_template::<Bls12_381>(c, black_box(n));
}
}
--
GitLab