Merge pull request #3421 from davidhewitt/codspeed

try adding codspeed benchmarks
PyO3 · Oct 20, 2023 · d8a6f37 · d8a6f37
2 parents 63e24df + bf2c5a8
commit d8a6f37
Show file tree

Hide file tree

Showing 19 changed files with 73 additions and 24 deletions.
diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml
@@ -0,0 +1,40 @@
+name: benches
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            pyo3-benches/target
+            target
+          key: cargo-${{ runner.os }}-bench-${{ hashFiles('**/Cargo.toml') }}
+        continue-on-error: true
+
+      - name: Install cargo-codspeed
+        run: cargo install cargo-codspeed
+
+      - name: Install nox
+        run: pip install nox
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@v1
+        with:
+          run: nox -s codspeed
+          token: ${{ secrets.CODSPEED_TOKEN }}
diff --git a/noxfile.py b/noxfile.py
@@ -137,6 +137,19 @@ def bench(session: nox.Session) -> bool:
     _run_cargo(session, "bench", _BENCHES, *session.posargs)
 
 
+@nox.session()
+def codspeed(session: nox.Session) -> bool:
+    # rust benchmarks
+    os.chdir(PYO3_DIR / "pyo3-benches")
+    _run_cargo(session, "codspeed", "build")
+    _run_cargo(session, "codspeed", "run")
+    # python benchmarks
+    os.chdir(PYO3_DIR / "pytests")
+    session.install("-r", "requirements-dev.txt", "pytest-codspeed")
+    session.install(".")
+    _run(session, "pytest", "--codspeed", external=True)
+
+
 @nox.session(name="clippy-all", venv_backend="none")
 def clippy_all(session: nox.Session) -> None:
     success = True

diff --git a/pyo3-benches/Cargo.toml b/pyo3-benches/Cargo.toml
@@ -7,11 +7,13 @@ edition = "2021"
 publish = false
 
 [dependencies]
-pyo3 = { path = "../", features = ["auto-initialize"] }
+pyo3 = { path = "../", features = ["auto-initialize", "full"] }
 
 [dev-dependencies]
+codspeed-criterion-compat = "2.3"
 criterion = "0.5.1"
 num-bigint = "0.4.3"
+rust_decimal = { version = "1.0.0", default-features = false }
 
 [[bench]]
 name = "bench_any"
@@ -32,17 +34,14 @@ harness = false
 [[bench]]
 name = "bench_decimal"
 harness = false
-required-features = ["pyo3/rust_decimal"]
 
 [[bench]]
 name = "bench_dict"
 harness = false
-required-features = ["pyo3/hashbrown"]
 
 [[bench]]
 name = "bench_frompyobject"
 harness = false
-required-features = ["pyo3/macros"]
 
 [[bench]]
 name = "bench_gil"
@@ -55,7 +54,6 @@ harness = false
 [[bench]]
 name = "bench_pyclass"
 harness = false
-required-features = ["pyo3/macros"]
 
 [[bench]]
 name = "bench_pyobject"
@@ -64,7 +62,6 @@ harness = false
 [[bench]]
 name = "bench_set"
 harness = false
-required-features = ["pyo3/hashbrown"]
 
 [[bench]]
 name = "bench_tuple"
@@ -81,6 +78,5 @@ harness = false
 [[bench]]
 name = "bench_bigint"
 harness = false
-required-features = ["pyo3/num-bigint"]
 
 [workspace]
diff --git a/pyo3-benches/benches/bench_any.rs b/pyo3-benches/benches/bench_any.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{
     types::{

diff --git a/pyo3-benches/benches/bench_bigint.rs b/pyo3-benches/benches/bench_bigint.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{types::PyDict, PyAny, Python};
 

diff --git a/pyo3-benches/benches/bench_call.rs b/pyo3-benches/benches/bench_call.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 

diff --git a/pyo3-benches/benches/bench_comparisons.rs b/pyo3-benches/benches/bench_comparisons.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{prelude::*, pyclass::CompareOp, Python};
 

diff --git a/pyo3-benches/benches/bench_decimal.rs b/pyo3-benches/benches/bench_decimal.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::PyDict;

diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::types::IntoPyDict;
 use pyo3::{prelude::*, types::PyMapping};

diff --git a/pyo3-benches/benches/bench_err.rs b/pyo3-benches/benches/bench_err.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{exceptions::PyValueError, prelude::*};
 

diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{
     types::{PyDict, PyFloat, PyInt, PyString},

diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{
     prelude::*,

diff --git a/pyo3-benches/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, BatchSize, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Bencher, Criterion};
 
 use pyo3::{prelude::*, GILPool};
 

diff --git a/pyo3-benches/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 

diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::{PyList, PySequence};

diff --git a/pyo3-benches/benches/bench_pyclass.rs b/pyo3-benches/benches/bench_pyclass.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 use pyo3::{impl_::pyclass::LazyTypeObject, prelude::*};
 
 /// This is a feature-rich class instance used to benchmark various parts of the pyclass lifecycle.
@@ -26,7 +26,7 @@ impl MyClass {
     }
 }
 
-pub fn first_time_init(b: &mut criterion::Bencher<'_>) {
+pub fn first_time_init(b: &mut Bencher<'_>) {
     Python::with_gil(|py| {
         b.iter(|| {
             // This is using an undocumented internal PyO3 API to measure pyclass performance; please

diff --git a/pyo3-benches/benches/bench_pyobject.rs b/pyo3-benches/benches/bench_pyobject.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 

diff --git a/pyo3-benches/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::PySet;

diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::{PyList, PySequence, PyTuple};