add builder metrics

Counters for some basic build metrics.
NixOS · May 28, 2020 · aa507e7 · aa507e7
1 parent 1e5d3ef
commit aa507e7
Show file tree

Hide file tree

Showing 5 changed files with 77 additions and 11 deletions.
diff --git a/ofborg/src/bin/builder.rs b/ofborg/src/bin/builder.rs
@@ -6,8 +6,7 @@ use async_std::task;
 use tracing::{info, warn};
 
 use ofborg::easyamqp::{self, ChannelExt, ConsumerExt};
-use ofborg::easylapin;
-use ofborg::{checkout, config, tasks};
+use ofborg::{checkout, config, easylapin, metrics, tasks};
 
 // FIXME: remove with rust/cargo update
 #[allow(clippy::cognitive_complexity)]
@@ -93,6 +92,8 @@ fn main() -> Result<(), Box<dyn Error>> {
         },
     )?;
 
+    metrics::spawn_server();
+
     info!("Fetching jobs from {}", &queue_name);
     task::block_on(handle);
 

diff --git a/ofborg/src/easylapin.rs b/ofborg/src/easylapin.rs
@@ -1,14 +1,5 @@
 use std::pin::Pin;
 
-use crate::config::RabbitMQConfig;
-use crate::easyamqp::{
-    BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType,
-    QueueConfig,
-};
-use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker};
-use crate::ofborg;
-use crate::worker::{Action, SimpleWorker};
-
 use async_std::future::Future;
 use async_std::stream::StreamExt;
 use async_std::task;
@@ -21,6 +12,16 @@ use lapin::types::{AMQPValue, FieldTable};
 use lapin::{BasicProperties, Channel, Connection, ConnectionProperties, ExchangeKind};
 use tracing::{debug, trace};
 
+use crate::config::RabbitMQConfig;
+use crate::easyamqp::{
+    BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType,
+    QueueConfig,
+};
+use crate::metrics;
+use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker};
+use crate::ofborg;
+use crate::worker::{Action, SimpleWorker};
+
 pub fn from_config(cfg: &RabbitMQConfig) -> Result<Connection, lapin::Error> {
     let mut props = FieldTable::default();
     props.insert(
@@ -166,6 +167,8 @@ impl<'a, W: SimpleNotifyWorker + 'a> ConsumerExt<'a, W> for NotifyChannel {
         Ok(Box::pin(async move {
             while let Some(Ok(deliver)) = consumer.next().await {
                 debug!(?deliver.delivery_tag, "consumed delivery");
+                metrics::JOBS_RECEIVED.inc();
+
                 let mut receiver = ChannelNotificationReceiver {
                     channel: &mut chan,
                     deliver: &deliver,

diff --git a/ofborg/src/message/buildresult.rs b/ofborg/src/message/buildresult.rs
@@ -11,6 +11,18 @@ pub enum BuildStatus {
     UnexpectedError { err: String },
 }
 
+impl BuildStatus {
+    pub fn as_label(&self) -> &'static str {
+        match self {
+            BuildStatus::Skipped => "skipped",
+            BuildStatus::Success => "success",
+            BuildStatus::Failure => "failure",
+            BuildStatus::TimedOut => "timeout",
+            BuildStatus::UnexpectedError { err: _ } => "unexpected",
+        }
+    }
+}
+
 impl From<BuildStatus> for String {
     fn from(status: BuildStatus) -> String {
         match status {

diff --git a/ofborg/src/metrics.rs b/ofborg/src/metrics.rs
@@ -12,6 +12,38 @@ use tracing::info;
 
 use crate::ofborg;
 
+lazy_static! {
+    pub static ref JOBS_RECEIVED: IntCounter = register_int_counter!(
+        "ofborg_jobs_received_total",
+        "Total number of jobs received."
+    )
+    .unwrap();
+    pub static ref BUILDS_RECEIVED: CounterVec = register_counter_vec!(
+        "ofborg_builds_received_total",
+        "Total number of builds received.",
+        &["system"]
+    )
+    .unwrap();
+    pub static ref BUILDS_FINISHED: CounterVec = register_counter_vec!(
+        "ofborg_builds_finished_total",
+        "Total number of builds finished.",
+        &["system", "status"]
+    )
+    .unwrap();
+    pub static ref BUILDS_ATTRIBUTES_ATTEMPTED: CounterVec = register_counter_vec!(
+        "ofborg_builds_attributes_attempted_total",
+        "Total number of attributes attempted to build.",
+        &["system"]
+    )
+    .unwrap();
+    pub static ref BUILDS_ATTRIBUTES_NOT_ATTEMPTED: CounterVec = register_counter_vec!(
+        "ofborg_builds_attributes_not_attempted_total",
+        "Total number of attributes not attempted to build.",
+        &["system"]
+    )
+    .unwrap();
+}
+
 lazy_static! {
     static ref VERSION: GaugeVec = register_gauge_vec!(
         "ofborg_version",

diff --git a/ofborg/src/tasks/build.rs b/ofborg/src/tasks/build.rs
@@ -2,6 +2,7 @@ use crate::checkout;
 use crate::commentparser;
 use crate::message::buildresult::{BuildResult, BuildStatus, V1Tag};
 use crate::message::{buildjob, buildlogmsg};
+use crate::metrics;
 use crate::nix;
 use crate::notifyworker;
 use crate::worker;
@@ -280,6 +281,9 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
     ) {
         let span = debug_span!("job", pr = ?job.pr.number);
         let _enter = span.enter();
+        metrics::BUILDS_RECEIVED
+            .with_label_values(&[&self.system])
+            .inc();
 
         let mut actions = self.actions(&job, notifier);
 
@@ -347,6 +351,14 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
             cannot_build_attrs.join(", ")
         );
 
+        metrics::BUILDS_ATTRIBUTES_ATTEMPTED
+            .with_label_values(&[&self.system])
+            .inc_by(can_build.len() as f64);
+
+        metrics::BUILDS_ATTRIBUTES_NOT_ATTEMPTED
+            .with_label_values(&[&self.system])
+            .inc_by(cannot_build_attrs.len() as f64);
+
         actions.log_started(can_build.clone(), cannot_build_attrs.clone());
         actions.log_instantiation_errors(cannot_build);
 
@@ -375,7 +387,13 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker {
             .last();
         info!("----->8-----");
 
+        let status_label = status.as_label();
         actions.build_finished(status, can_build, cannot_build_attrs);
+
+        metrics::BUILDS_FINISHED
+            .with_label_values(&[&self.system, status_label])
+            .inc();
+
         info!("Build done!");
     }
 }