feat(ml-service): add detailed ML observability metrics for inference, predictions, and errors

apps/ml-service/src/routes.rs

@@ -130,18 +130,33 @@ pub async fn predict(
     };
 
     // Preprocess the image
+    let preprocess_start = std::time::Instant::now();
     let input = preprocess_image(&bytes, state.model.input_size())?;
 
-    // Run prediction
-    let prediction = state.model.predict(input)?;
+    // Record image size metric
+    telemetry::image_size_bytes().observe(bytes.len() as f64);
 
-    // Record business and request telemetry
+    // Run prediction with timing
+    let inference_start = std::time::Instant::now();
+    let prediction = state.model.predict(input)?;
+    telemetry::inference_duration_seconds().observe(inference_start.elapsed().as_secs_f64());
+
+    // Record business telemetry
     telemetry::predictions_total().inc();
+    telemetry::predictions_by_class()
+        .with_label_values(&[&prediction.label])
+        .inc();
+    telemetry::predictions_confidence().observe(prediction.confidence as f64);
     _guard.finish();
 
     Ok(Json(prediction_response(prediction)))
 }
 
+/// Helper to record errors from route handlers
+pub fn record_error(kind: &str) {
+    telemetry::errors_total().with_label_values(&[kind]).inc();
+}
+
 pub fn router(state: AppState) -> Router {
     Router::new()
         .route("/health", get(health))

apps/ml-service/src/telemetry.rs

@@ -1,4 +1,4 @@
-use prometheus::{Counter, Gauge, Histogram, HistogramOpts, Registry, TextEncoder};
+use prometheus::{Counter, CounterVec, Gauge, Histogram, HistogramOpts, HistogramVec, Opts, Registry, TextEncoder};
 use std::sync::OnceLock;
 use std::time::Instant;
 
@@ -78,6 +78,76 @@ define_metric!(
     .expect("create gauge")
 );
 
+/// Per-class prediction counter
+define_metric!(
+    predictions_by_class,
+    CounterVec,
+    CounterVec::new(
+        Opts::new(
+            "zeavis_ml_predictions_by_class_total",
+            "Total predictions by predicted class label",
+        ),
+        &["label"],
+    )
+    .expect("create counter_vec")
+);
+
+/// Per-class ground-truth counter (for monitoring label distribution)
+define_metric!(
+    predictions_confidence,
+    Histogram,
+    Histogram::with_opts(
+        HistogramOpts::new(
+            "zeavis_ml_prediction_confidence",
+            "Confidence values of predictions",
+        )
+        .buckets(vec![0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 0.99, 1.0]),
+    )
+    .expect("create histogram")
+);
+
+/// Latency of ONNX inference (model.predict call)
+define_metric!(
+    inference_duration_seconds,
+    Histogram,
+    Histogram::with_opts(
+        HistogramOpts::new(
+            "zeavis_ml_inference_duration_seconds",
+            "ONNX model inference duration in seconds",
+        )
+        .buckets(vec![0.01, 0.025, 0.05, 0.1, 0.2, 0.3, 0.5, 0.75, 1.0, 2.0]),
+    )
+    .expect("create histogram")
+);
+
+/// Image size processed by the ML service
+define_metric!(
+    image_size_bytes,
+    Histogram,
+    Histogram::with_opts(
+        HistogramOpts::new(
+            "zeavis_ml_image_size_bytes",
+            "Size of images sent for prediction in bytes",
+        )
+        .buckets(vec![1024.0, 10240.0, 51200.0, 102400.0, 204800.0, 512000.0, 1048576.0, 2097152.0]),
+    )
+    .expect("create histogram")
+);
+
+/// Error counter by error kind (e.g. bad_request, model_error, internal)
+define_metric!(
+    errors_total,
+    CounterVec,
+    CounterVec::new(
+        Opts::new(
+            "zeavis_ml_errors_total",
+            "Total errors by kind",
+        ),
+        &["kind"],
+    )
+    .expect("create counter_vec")
+);
+
 // ── Request Guard (Drop-based cleanup for active gauge) ─
 
 pub struct RequestMetricsGuard {