Fix benches

CONTRIBUTING.rst

@@ -75,7 +75,11 @@ Multiple builds with misc. trade-offs can be used, depending on what you are wor
 - When developping tests, you can do the same as for Python development, but
   using the ``test`` environment to run the test faster!
 
+- Running benchmarks (those are implemented in ``src/scalib_ext/scalib/benches``:
 
+.. code-block::
+
+   make bench
 
 Before committing or pull request
 ---------------------------------

Makefile

@@ -12,6 +12,7 @@ help:
 	@echo "    docs: Generate docs."
 	@echo "    codestyle: Check fromatting."
 	@echo "    fmt: Format code."
+	@echo "    bench: Run Rust benchmarks (assuming x86_64 v3 level CPU)."
 	@echo "    wheel_local: Build a wheel for the local machine."
 	@echo "    wheel_portable: Build a wheel for with maximal portability (at the expense of efficiency)."
 	@echo "    wheel_x86_64_v3: Build a wheel for X86_64 v3 level CPUs."
@@ -49,3 +50,6 @@ wheel_local:
 
 wheel_portable:
 	SCALIB_PORTABLE=1 CARGO_TARGET_DIR=.cargo_build_portable  pyproject-build -w -o dist/portable
+
+bench:
+	cd src/scalib_ext/scalib; RUSTFLAGS="-C target-cpu=x86-64-v3" CARGO_TARGET_DIR=../target_bench cargo bench

src/scalib/config/__init__.py

@@ -105,11 +105,10 @@ def __init__(
         n_threads: Optional[int] = None,
         show_progress: Optional[bool] = None,
     ):
-        self.inner = _scalib_ext.Config()
         if show_progress is not None:
-            self.inner.show_progress(show_progress)
+            self.inner = _scalib_ext.Config(show_progress)
         else:
-            self.inner.show_progress(get_config().inner.progress())
+            self.inner = get_config().inner
         if threadpool is not None:
             self.threadpool = threadpool
         elif n_threads is not None:

src/scalib_ext/scalib-py/src/lib.rs

@@ -42,17 +42,15 @@ pub struct Config {
 #[pymethods]
 impl Config {
     #[new]
-    fn new() -> Self {
+    fn new(show_progress: bool) -> Self {
         Self {
-            inner: Default::default(),
+            inner: if show_progress {
+                scalib::Config::with_default_timing()
+            } else {
+                scalib::Config::no_progress()
+            },
         }
     }
-    fn show_progress(&mut self, show: bool) {
-        self.inner.show_progress = show;
-    }
-    fn progress(&self) -> bool {
-        self.inner.show_progress
-    }
 }
 
 #[derive(FromPyObject)]

src/scalib_ext/scalib/Cargo.toml

@@ -75,11 +75,7 @@ name = "belief_propagation"
 harness = false
 
 [[bench]]
-name = "get_snr"
-harness = false
-
-[[bench]]
-name = "snr_update"
+name = "snr"
 harness = false
 
 [[bench]]

src/scalib_ext/scalib/benches/belief_propagation.rs

@@ -1,6 +1,4 @@
-use criterion::{
-    criterion_group, criterion_main, AxisScale, BenchmarkId, Criterion, PlotConfiguration,
-};
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
 use ndarray::{s, Array1, Array2};
 use std::fmt;
 #[inline(always)]
@@ -103,7 +101,8 @@ fn xors_bench(c: &mut Criterion) {
     for ni in [2].iter() {
         let id = fmt::format(format_args!("n_inputs_{}", *ni));
         let mut group = c.benchmark_group(id);
-        for nc in [2, 4, 8, 16, 32, 64, 128, 256].iter() {
+        //for nc in [2, 4, 8, 16, 32, 64, 128, 256].iter() {
+        for nc in [2].iter() {
             group.bench_with_input(BenchmarkId::new("xors", nc), nc, |b, nc| {
                 b.iter(|| {
                     let mut inputs: Vec<Array2<f64>> = (0..*ni as usize)

src/scalib_ext/scalib/benches/mttest.rs

@@ -3,11 +3,9 @@ use ndarray::{Array1, Array2};
 use ndarray_rand::rand_distr::Uniform;
 use ndarray_rand::RandomExt;
 use scalib::mttest;
-use std::time::Duration;
 
 fn bench_mttest(c: &mut Criterion) {
     let mut group = c.benchmark_group("ttest_update");
-    let d = 2;
     let traces_len = 1000;
     let n = 2048;
     let traces = Array2::<i16>::random((n, traces_len), Uniform::new(0, 1000));
@@ -21,7 +19,7 @@ fn bench_mttest(c: &mut Criterion) {
             group.bench_with_input(
                 BenchmarkId::new(format!("mttest_{}", *d), npois),
                 npois,
-                |b, npois| {
+                |b, _npois| {
                     b.iter(|| {
                         mtt.update(traces.view(), y.view());
                     })
@@ -35,7 +33,7 @@ fn bench_mttest(c: &mut Criterion) {
 criterion_group! {
     name = benches;
     // This can be any expression that returns a `Criterion` object.
-    config = Criterion::default().significance_level(0.01).sample_size(500).measurement_time(Duration::from_secs(60));
+    config = Criterion::default().sample_size(50);
     targets = bench_mttest
 }
 criterion_main!(benches);

src/scalib_ext/scalib/benches/snr.rs

@@ -0,0 +1,101 @@
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use ndarray::Array2;
+use ndarray_rand::rand_distr::Uniform;
+use ndarray_rand::RandomExt;
+use scalib::{snr, Config};
+
+type BenchMarkGroup<'a> = criterion::BenchmarkGroup<'a, criterion::measurement::WallTime>;
+
+fn gen_traces(n: usize, ns: usize) -> Array2<i16> {
+    Array2::<i16>::random((n, ns), Uniform::new(0, 100))
+}
+fn gen_classes(np: usize, n: usize, nc: usize) -> Array2<u16> {
+    Array2::<u16>::random((np, n), Uniform::new(0, nc as u16))
+}
+
+fn bench_get_snr_inner<S: snr::SnrType<Sample = i16>>(
+    nc: usize,
+    np: usize,
+    ns: usize,
+    n: usize,
+    group: &mut BenchMarkGroup,
+) {
+    let x = gen_traces(n, ns);
+    let y = gen_classes(np, n, nc);
+    let mut snr = snr::SNR::<S>::new(nc, ns, np);
+    snr.update(x.view(), y.view(), &Config::no_progress())
+        .unwrap();
+    group.bench_with_input(BenchmarkId::new("get_snr", ns), &ns, |b, _| {
+        b.iter(|| {
+            snr.get_snr();
+        })
+    });
+}
+
+fn bench_snr_update_inner<S: snr::SnrType<Sample = i16>>(
+    nc: usize,
+    np: usize,
+    ns: usize,
+    n: usize,
+    group: &mut BenchMarkGroup,
+) {
+    let x = gen_traces(n, ns);
+    let y = gen_classes(np, n, nc);
+    let mut snr = snr::SNR::<S>::new(nc, ns, np);
+    group.bench_with_input(
+        BenchmarkId::new(format!("chunk_{}", np), ns),
+        &ns,
+        |b, _| {
+            b.iter(|| {
+                snr.update(x.view(), y.view(), &Config::no_progress())
+                    .unwrap();
+            })
+        },
+    );
+}
+
+fn bench_get_snr(c: &mut Criterion) {
+    let nc = 256;
+    let np = 16;
+    let n = 1000;
+    for i in [32, 64] {
+        let mut group = c.benchmark_group(format!("get_snr_{}", i));
+        for ns in [1000, 100000] {
+            if i == 32 {
+                bench_get_snr_inner::<snr::SnrType32bit>(nc, np, ns, n, &mut group);
+            }
+            if i == 64 {
+                bench_get_snr_inner::<snr::SnrType64bit>(nc, np, ns, n, &mut group);
+            }
+        }
+        group.finish();
+    }
+}
+
+fn bench_snr_update(c: &mut Criterion) {
+    let nc = 256;
+    let n = 10000;
+
+    for i in [32, 64] {
+        let mut group = c.benchmark_group(format!("snr_update_{}", i));
+        for ns in [1 << 10, 1 << 16] {
+            for np in [1, 16] {
+                if i == 32 {
+                    bench_snr_update_inner::<snr::SnrType32bit>(nc, np, ns, n, &mut group);
+                }
+                if i == 64 {
+                    bench_snr_update_inner::<snr::SnrType64bit>(nc, np, ns, n, &mut group);
+                }
+            }
+        }
+        group.finish();
+    }
+}
+
+criterion_group! {
+    name = benches;
+    // This can be any expression that returns a `Criterion` object.
+    config = Criterion::default().significance_level(0.1).sample_size(10);
+    targets = bench_get_snr, bench_snr_update
+}
+criterion_main!(benches);

src/scalib_ext/scalib/benches/ttest.rs

@@ -9,7 +9,7 @@ fn bench_mttest(c: &mut Criterion) {
     let mut group = c.benchmark_group("ttest_update");
     let n = 5000;
     for d in [2, 3].iter() {
-        for traces_len in [5000, 10000, 20000].iter() {
+        for traces_len in [20000].iter() {
             let traces = Array2::<i16>::random((n, *traces_len), Uniform::new(0, 1000));
             let y = Array1::<u16>::random((n,), Uniform::new(0, 2));
 
@@ -18,7 +18,7 @@ fn bench_mttest(c: &mut Criterion) {
             group.bench_with_input(
                 BenchmarkId::new(format!("ttest_{}", traces_len), *d),
                 d,
-                |b, d| {
+                |b, _d| {
                     b.iter(|| {
                         tt.update(traces.view(), y.view());
                     })
@@ -32,7 +32,7 @@ fn bench_mttest(c: &mut Criterion) {
 criterion_group! {
     name = benches;
     // This can be any expression that returns a `Criterion` object.
-    config = Criterion::default().significance_level(0.01).sample_size(500).measurement_time(Duration::from_secs(30));
+    config = Criterion::default().sample_size(50);
     targets = bench_mttest
 }
 criterion_main!(benches);

src/scalib_ext/scalib/src/lib.rs

@@ -39,20 +39,23 @@ pub enum ScalibError {
     NoAssociatedClassesStored,
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct Config {
-    /// Show progress bars
-    pub show_progress: bool,
     /// Computation time after which a progress bar is displayed.
     /// This avoids showing progress bars for negligible amounts of time.
-    pub progress_min_time: std::time::Duration,
+    /// If None, never display the progress bar
+    progress_min_time: Option<std::time::Duration>,
 }
 
-impl std::default::Default for Config {
-    fn default() -> Self {
+impl Config {
+    pub fn with_default_timing() -> Self {
         Self {
-            show_progress: true,
-            progress_min_time: std::time::Duration::from_millis(500),
+            progress_min_time: Some(std::time::Duration::from_millis(500)),
+        }
+    }
+    pub fn no_progress() -> Self {
+        Self {
+            progress_min_time: None,
         }
     }
 }