From 35bfc7321eac61089785615c8105c0be29ddeed2 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Wed, 27 May 2020 23:47:27 +0200 Subject: [PATCH] add builder metrics Counters for some basic build metrics. --- ofborg/src/bin/builder.rs | 5 +++-- ofborg/src/easylapin.rs | 21 +++++++++++--------- ofborg/src/message/buildresult.rs | 12 ++++++++++++ ofborg/src/metrics.rs | 32 +++++++++++++++++++++++++++++++ ofborg/src/tasks/build.rs | 18 +++++++++++++++++ 5 files changed, 77 insertions(+), 11 deletions(-) diff --git a/ofborg/src/bin/builder.rs b/ofborg/src/bin/builder.rs index 2a22c798..b857ee16 100644 --- a/ofborg/src/bin/builder.rs +++ b/ofborg/src/bin/builder.rs @@ -6,8 +6,7 @@ use async_std::task; use tracing::{info, warn}; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::{checkout, config, tasks}; +use ofborg::{checkout, config, easylapin, metrics, tasks}; // FIXME: remove with rust/cargo update #[allow(clippy::cognitive_complexity)] @@ -93,6 +92,8 @@ fn main() -> Result<(), Box> { }, )?; + metrics::spawn_server(); + info!("Fetching jobs from {}", &queue_name); task::block_on(handle); diff --git a/ofborg/src/easylapin.rs b/ofborg/src/easylapin.rs index c74ce6ad..0008eac7 100644 --- a/ofborg/src/easylapin.rs +++ b/ofborg/src/easylapin.rs @@ -1,14 +1,5 @@ use std::pin::Pin; -use crate::config::RabbitMQConfig; -use crate::easyamqp::{ - BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType, - QueueConfig, -}; -use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker}; -use crate::ofborg; -use crate::worker::{Action, SimpleWorker}; - use async_std::future::Future; use async_std::stream::StreamExt; use async_std::task; @@ -21,6 +12,16 @@ use lapin::types::{AMQPValue, FieldTable}; use lapin::{BasicProperties, Channel, Connection, ConnectionProperties, ExchangeKind}; use tracing::{debug, trace}; +use crate::config::RabbitMQConfig; +use crate::easyamqp::{ + BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType, + QueueConfig, +}; +use crate::metrics; +use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker}; +use crate::ofborg; +use crate::worker::{Action, SimpleWorker}; + pub fn from_config(cfg: &RabbitMQConfig) -> Result { let mut props = FieldTable::default(); props.insert( @@ -166,6 +167,8 @@ impl<'a, W: SimpleNotifyWorker + 'a> ConsumerExt<'a, W> for NotifyChannel { Ok(Box::pin(async move { while let Some(Ok(deliver)) = consumer.next().await { debug!(?deliver.delivery_tag, "consumed delivery"); + metrics::JOBS_RECEIVED.inc(); + let mut receiver = ChannelNotificationReceiver { channel: &mut chan, deliver: &deliver, diff --git a/ofborg/src/message/buildresult.rs b/ofborg/src/message/buildresult.rs index a292edea..a55db110 100644 --- a/ofborg/src/message/buildresult.rs +++ b/ofborg/src/message/buildresult.rs @@ -11,6 +11,18 @@ pub enum BuildStatus { UnexpectedError { err: String }, } +impl BuildStatus { + pub fn as_label(&self) -> &'static str { + match self { + BuildStatus::Skipped => "skipped", + BuildStatus::Success => "success", + BuildStatus::Failure => "failure", + BuildStatus::TimedOut => "timeout", + BuildStatus::UnexpectedError { err: _ } => "unexpected", + } + } +} + impl From for String { fn from(status: BuildStatus) -> String { match status { diff --git a/ofborg/src/metrics.rs b/ofborg/src/metrics.rs index 2e255977..b7d73a94 100644 --- a/ofborg/src/metrics.rs +++ b/ofborg/src/metrics.rs @@ -12,6 +12,38 @@ use tracing::info; use crate::ofborg; +lazy_static! { + pub static ref JOBS_RECEIVED: IntCounter = register_int_counter!( + "ofborg_jobs_received_total", + "Total number of jobs received." + ) + .unwrap(); + pub static ref BUILDS_RECEIVED: CounterVec = register_counter_vec!( + "ofborg_builds_received_total", + "Total number of builds received.", + &["system"] + ) + .unwrap(); + pub static ref BUILDS_FINISHED: CounterVec = register_counter_vec!( + "ofborg_builds_finished_total", + "Total number of builds finished.", + &["system", "status"] + ) + .unwrap(); + pub static ref BUILDS_ATTRIBUTES_ATTEMPTED: CounterVec = register_counter_vec!( + "ofborg_builds_attributes_attempted_total", + "Total number of attributes attempted to build.", + &["system"] + ) + .unwrap(); + pub static ref BUILDS_ATTRIBUTES_NOT_ATTEMPTED: CounterVec = register_counter_vec!( + "ofborg_builds_attributes_not_attempted_total", + "Total number of attributes not attempted to build.", + &["system"] + ) + .unwrap(); +} + lazy_static! { static ref VERSION: GaugeVec = register_gauge_vec!( "ofborg_version", diff --git a/ofborg/src/tasks/build.rs b/ofborg/src/tasks/build.rs index e4f32bd9..380ccc5c 100644 --- a/ofborg/src/tasks/build.rs +++ b/ofborg/src/tasks/build.rs @@ -2,6 +2,7 @@ use crate::checkout; use crate::commentparser; use crate::message::buildresult::{BuildResult, BuildStatus, V1Tag}; use crate::message::{buildjob, buildlogmsg}; +use crate::metrics; use crate::nix; use crate::notifyworker; use crate::worker; @@ -280,6 +281,9 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker { ) { let span = debug_span!("job", pr = ?job.pr.number); let _enter = span.enter(); + metrics::BUILDS_RECEIVED + .with_label_values(&[&self.system]) + .inc(); let mut actions = self.actions(&job, notifier); @@ -347,6 +351,14 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker { cannot_build_attrs.join(", ") ); + metrics::BUILDS_ATTRIBUTES_ATTEMPTED + .with_label_values(&[&self.system]) + .inc_by(can_build.len() as f64); + + metrics::BUILDS_ATTRIBUTES_NOT_ATTEMPTED + .with_label_values(&[&self.system]) + .inc_by(cannot_build_attrs.len() as f64); + actions.log_started(can_build.clone(), cannot_build_attrs.clone()); actions.log_instantiation_errors(cannot_build); @@ -375,7 +387,13 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker { .last(); info!("----->8-----"); + let status_label = status.as_label(); actions.build_finished(status, can_build, cannot_build_attrs); + + metrics::BUILDS_FINISHED + .with_label_values(&[&self.system, status_label]) + .inc(); + info!("Build done!"); } }