Merge branch 'main' of https://github.com/ATLAS-PJK-GM007/ZeaVis-Edu into selly/frontend

This commit is contained in:
seriouselly
2026-06-08 17:21:27 +07:00
43 changed files with 2352 additions and 30 deletions
+6
View File
@@ -1,3 +1,9 @@
WEB_PORT=5173
API_PORT=3000
DATABASE_URL=postgres://postgres:postgres@localhost:5432/zeavis_edu
# ── Telemetry / ClickHouse ──────────────────────────────────────────
# These credentials are used by the telemetry Docker Compose stack.
# See telemetry/deploy/.env.example for production overrides.
CLICKHOUSE_USER=telemetry
CLICKHOUSE_PASSWORD=telemetry
+6 -2
View File
@@ -118,8 +118,8 @@ jobs:
fi
cd "$DEPLOY_PATH"
git checkout main
git pull --ff-only origin main
git fetch origin main
git reset --hard origin/main
{
printf 'GITHUB_REPOSITORY=%s\n' "$REPO_SLUG"
@@ -133,9 +133,13 @@ jobs:
docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
docker network create app-shared-net 2>/dev/null || true
docker network create telemetry-net 2>/dev/null || true
docker compose down --remove-orphans || true
docker rm -f zeavis-web zeavis-api zeavis-ml zeavis-node-exporter 2>/dev/null || true
docker compose pull
docker compose up -d
docker compose ps
docker compose ps | grep -q "zeavis-web.*Up" || exit 1
docker compose ps | grep -q "zeavis-api.*Up" || exit 1
docker compose ps | grep -q "zeavis-ml.*Up" || exit 1
docker compose ps | grep -q "zeavis-node-exporter.*Up" || echo "⚠️ node_exporter not running (non-fatal)"
+241
View File
@@ -0,0 +1,241 @@
name: Telemetry CI/CD — Build & Deploy to Orange VPS
on:
push:
branches: [main, master]
paths:
- 'telemetry/**'
- '.github/workflows/telemetry-ci-cd.yml'
workflow_dispatch:
env:
GHCR_REGISTRY: ghcr.io
METRIC_INGESTER_IMAGE: mytheclipse/telemetry-metric-ingester
QUERY_PROXY_IMAGE: mytheclipse/telemetry-query-proxy
jobs:
##############################################################################
# BUILD: Metric Ingester (Go service)
##############################################################################
build-metric-ingester:
name: Build - Metric Ingester
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ${{ env.GHCR_REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GHCR_PAT }}
- name: Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.GHCR_REGISTRY }}/${{ env.METRIC_INGESTER_IMAGE }}
tags: |
type=sha,prefix=,suffix=,format=short
- name: Build & push
uses: docker/build-push-action@v6
with:
push: true
file: telemetry/deploy/Dockerfile.metric-ingester
context: telemetry
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
##############################################################################
# BUILD: Query Proxy (Go service)
##############################################################################
build-query-proxy:
name: Build - Query Proxy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ${{ env.GHCR_REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GHCR_PAT }}
- name: Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.GHCR_REGISTRY }}/${{ env.QUERY_PROXY_IMAGE }}
tags: |
type=sha,prefix=,suffix=,format=short
- name: Build & push
uses: docker/build-push-action@v6
with:
push: true
file: telemetry/deploy/Dockerfile.query-proxy
context: telemetry
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
##############################################################################
# BUILD: Telemetry UI (Vue 3 SPA build verification)
##############################################################################
build-telemetry-ui:
name: Build - Telemetry UI SPA
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- uses: actions/setup-node@v4
with:
node-version: '22'
cache: 'npm'
cache-dependency-path: telemetry/telemetry-ui/package-lock.json
- run: cd telemetry/telemetry-ui && npm ci
- run: cd telemetry/telemetry-ui && npm run build
- run: echo "Telemetry UI SPA built and verified."
##############################################################################
# DEPLOY: to Orange VPS (100.96.248.86)
##############################################################################
deploy:
name: Deploy to Orange VPS
needs:
- build-metric-ingester
- build-query-proxy
- build-telemetry-ui
runs-on: ubuntu-latest
if: |
always()
&& !cancelled()
&& !contains(needs.*.result, 'failure')
&& (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: git short-sha
id: sha
run: echo "sha=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
- name: Build Telemetry UI SPA
working-directory: telemetry/telemetry-ui
run: |
npm ci
npm run build
- name: Deploy via SSH
env:
VPS_HOST: ${{ secrets.TELEMETRY_VPS_HOST }}
VPS_PORT: ${{ secrets.TELEMETRY_VPS_PORT || '22' }}
VPS_USER: ${{ secrets.TELEMETRY_VPS_USER }}
GHCR_PAT: ${{ secrets.GHCR_PAT }}
SHA: ${{ steps.sha.outputs.sha }}
run: |
# Setup SSH key
mkdir -p ~/.ssh
echo "${{ secrets.TELEMETRY_VPS_SSH_KEY }}" > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
ssh-keyscan -p "$VPS_PORT" "$VPS_HOST" >> ~/.ssh/known_hosts 2>/dev/null
# Prepare directories on VPS
ssh -p "$VPS_PORT" "$VPS_USER@$VPS_HOST" "mkdir -p /opt/telemetry"
# Sync config files
rsync -avz --delete -e "ssh -p $VPS_PORT" \
telemetry/prometheus/ \
"$VPS_USER@$VPS_HOST:/opt/telemetry/prometheus/"
rsync -avz --delete -e "ssh -p $VPS_PORT" \
telemetry/metric-ingester/ \
"$VPS_USER@$VPS_HOST:/opt/telemetry/metric-ingester/"
rsync -avz --delete -e "ssh -p $VPS_PORT" \
telemetry/vector/ \
"$VPS_USER@$VPS_HOST:/opt/telemetry/vector/"
rsync -avz --delete -e "ssh -p $VPS_PORT" \
telemetry/clickhouse/ \
"$VPS_USER@$VPS_HOST:/opt/telemetry/clickhouse/"
rsync -avz --delete -e "ssh -p $VPS_PORT" \
telemetry/query-proxy/ \
"$VPS_USER@$VPS_HOST:/opt/telemetry/query-proxy/"
rsync -avz --delete -e "ssh -p $VPS_PORT" \
telemetry/telemetry-ui/dist/ \
"$VPS_USER@$VPS_HOST:/opt/telemetry/telemetry-ui/dist/"
rsync -avz --delete -e "ssh -p $VPS_PORT" \
telemetry/deploy/nginx/ \
"$VPS_USER@$VPS_HOST:/opt/telemetry/nginx/"
# Copy and prepare docker-compose.yml
scp -P "$VPS_PORT" telemetry/deploy/docker-compose.yml \
"$VPS_USER@$VPS_HOST:/opt/telemetry/docker-compose.yml"
# Run deploy on VPS
ssh -p "$VPS_PORT" "$VPS_USER@$VPS_HOST" bash -s << 'DEPLOYEOF'
set -e
echo '=== Telemetry Deploy to Orange VPS ==='
cd /opt/telemetry
# Create .env with secrets (injected by GHA)
cat > .env << ENVEOF
CLICKHOUSE_USER=${CLICKHOUSE_USER:-telemetry}
CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-telemetry}
ENVEOF
# Login to GHCR
echo '${{ secrets.GHCR_PAT }}' | docker login ghcr.io -u '${{ github.actor }}' --password-stdin
# Rewrite docker-compose to use SHA-tagged images
SHA='${{ steps.sha.outputs.sha }}'
TMP_FILE=$(mktemp /tmp/telemetry-compose-XXXXXX)
awk -v sha="$SHA" '
/^[[:space:]]*image:[[:space:]]+ghcr.io\/[Mm]ythe?[Ee]clipse\/telemetry-/ {
sub(/:([^:]*)$/, ":" sha)
}
{ print }
' docker-compose.yml > "$TMP_FILE"
mv "$TMP_FILE" docker-compose.yml
# Pull SHA-tagged images
docker pull "ghcr.io/mytheclipse/telemetry-metric-ingester:$SHA" || true
docker pull "ghcr.io/mytheclipse/telemetry-query-proxy:$SHA" || true
# Tear down old stack (preserve volumes)
docker compose down --remove-orphans --volumes=false 2>/dev/null || true
# Deploy fresh
docker compose up -d --remove-orphans --no-build
sleep 5
# Run ClickHouse schema migration
echo '=== Running ClickHouse schema migration ==='
bash clickhouse/init.sh 2>&1 || echo '⚠️ Schema migration had issues (non-fatal)'
echo '=== Schema migration complete ==='
docker compose ps
echo '=== Deploy Complete ==='
DEPLOYEOF
+2
View File
@@ -11,3 +11,5 @@ coverage/
*.tsbuildinfo
.DS_Store
.claude/
+3
View File
@@ -0,0 +1,3 @@
[submodule "telemetry"]
path = telemetry
url = https://github.com/MythEclipse/Telemetry.git
+41
View File
@@ -107,6 +107,31 @@ Run the ML service directly:
cd apps/ml-service && cargo run
```
Run the Telemetry stack:
```bash
# Start all telemetry services (Prometheus, Ingester, Vector, ClickHouse, Query Proxy, Telemetry UI)
make telemetry-up
# Local dev mode (port bindings exposed)
make telemetry-up-local
# Check health of all telemetry services
make telemetry-status
# View telemetry logs
make telemetry-logs [s=<service>]
# Build telemetry components
make telemetry-build
# Send a test metric
make telemetry-test-metric
# Stop telemetry
make telemetry-down
```
## High-level architecture
- `Machine_Learning/preprocessing.py` prepares the training dataset locally. It extracts three source ZIP files, merges selected class folders into `dataset/`, maps selected Mandarin labels from Dataset 3 via `desc.json`, removes known problematic image files, then creates `dataset.zip` for upload to Google Drive/Colab.
@@ -116,6 +141,22 @@ cd apps/ml-service && cargo run
- TensorFlow.js export is intentionally done with the `tensorflowjs_converter` CLI rather than from Python to avoid protobuf/runtime conflicts documented in the README.
- `apps/ml-service/` is a Rust/Axum service that loads the ONNX model and serves HTTP endpoints for health checks, metadata, and image classification predictions. It uses ONNX Runtime for cross-platform inference performance.
## Telemetry architecture
The repository includes a full Prometheus → ClickHouse metric pipeline as a git submodule at `telemetry/`. Each ZeaVis Edu service exposes a `GET /metrics` endpoint:
- **Web app** (`apps/web`): In dev mode, a Vite plugin serves client-side session metrics (page views, Web Vitals). In production, nginx proxies `/metrics` to the API service. Source: `apps/web/src/lib/telemetry.ts`, `apps/web/vite-plugin-metrics.ts`.
- **API** (`apps/api`): Uses `prom-client` for Node.js default metrics plus custom HTTP, auth, classification, and diagnosis counters/histograms. Source: `apps/api/src/lib/telemetry.ts`, exposed via `apps/api/src/routes/metrics.ts`.
- **ML service** (`apps/ml-service`): Uses the `prometheus` Rust crate for HTTP metrics, prediction counts, and model load status. Source: `apps/ml-service/src/telemetry.rs`.
All three share the `zeavis_` metric prefix and are scraped by the Telemetry Prometheus instance via `file_sd_configs` (see `telemetry/prometheus/targets/zeavis-edu.json`).
**IMPORTANT — Production architecture:** ZeaVis Edu apps and the Telemetry stack run on **separate VPS instances** connected via **Tailscale** (mesh VPN). Prometheus scrapes the API and ML service through their **Tailscale IPs** (e.g. `100.x.x.a:3000`), not via Docker hostnames. The target file `telemetry/prometheus/targets/zeavis-edu.json` has `__CHANGE_ME__` placeholders — before deploying, replace with the actual Tailscale IPs of the app VPS.
The telemetry stack is managed from the project root via `make telemetry-*` targets (see `Makefile`). The Docker Compose files in `telemetry/deploy/` define 6 services (Prometheus, Metric Ingester, Vector, ClickHouse, Query Proxy, Telemetry UI).
For **local single-host dev**, Prometheus can reach app services via a shared Docker network (`app-shared-net`). Use `make telemetry-up-local` for this mode — it includes the `docker-compose.telemetry.yml` override.
## Fullstack application architecture
The root TypeScript workspace is a Bun + Moon monorepo:
+121
View File
@@ -0,0 +1,121 @@
# ZeaVis Edu — Metrics Endpoints
This document lists every Prometheus metrics endpoint exposed by the ZeaVis Edu
application stack and the payload each service provides.
---
## Overview
| Service | Host (prod) | Metrics Endpoint | Port (local) |
|-----------------------|-----------------------------------|----------------------------|--------------|
| Web (Vite dev) | `zeavisedu.asepharyana.my.id` | `GET /metrics` | 5173 |
| API (Elysia) | `api-zeavisedu.asepharyana.my.id` | `GET /metrics` | 3000 |
| ML Service (Axum) | `ml-zeavisedu.asepharyana.my.id` | `GET /metrics` | 8000 |
| Prometheus Collector | — | `GET /metrics` (self) | 9090 |
> In production all metrics are scraped by the Prometheus collector running in the
> Telemetry stack on a **separate VPS** connected via **Tailscale**.
> See [`telemetry/prometheus/targets/`](./telemetry/prometheus/targets/)
> for the auto‑discovery configuration. Target files must use **Tailscale IPs**
> (e.g. `100.x.x.a:3000`), not Docker hostnames, because the services are on
> different hosts.
>
> In production (nginx), the web app proxies `/metrics` to the API service:
> see [`apps/web/nginx.conf`](apps/web/nginx.conf).
>
> For local development the Vite plugin `vite-plugin-metrics.ts` serves
> client‑side session metrics at `GET /metrics` on the Vite dev server.
---
## 1. Web App — `GET /metrics`
| Endpoint | Description |
|-------------------|--------------------------------------------------|
| `/metrics` | Vite dev‑server middleware + client‑side snapshot |
### Metrics
| Metric Name | Type | Labels | Description |
|-------------------------------------|---------|-------------------------------|------------------------------------------|
| `zeavis_web_page_views_total` | counter | — | Total page views this session |
| `zeavis_web_vital_bucket` | gauge | `name`, `rating` | Last‑seen Web Vitals (CLS, FCP, INP…) |
**Development:** served inline by the Vite plugin `vite-plugin-metrics.ts`.
**Production:** the static frontend serves no `/metrics` endpoint — consider
forwarding the Vite dev server, or use the Telemetry collector to scrape
client‑side beacons.
---
## 2. API (Elysia/Bun) — `GET /metrics`
| Endpoint | Description |
|-------------------|--------------------------------------------------|
| `/metrics` | Prometheus text format via `prom-client` |
### Metrics
| Metric Name | Type | Labels | Description |
|--------------------------------------------|-----------|--------------------------------|------------------------------------------|
| `zeavis_api_http_requests_total` | counter | `method`, `path`, `status` | Total HTTP requests |
| `zeavis_api_http_request_duration_seconds` | histogram | `method`, `path` | Request latency buckets |
| `zeavis_api_http_requests_active` | gauge | — | Concurrently‑handled requests |
| `zeavis_api_classifications_total` | counter | `result` | AI image classifications |
| `zeavis_api_diagnoses_total` | counter | `disease` | Created diagnoses |
| `zeavis_api_auth_operations_total` | counter | `operation`, `success` | Login / register attempts |
| Default Node.js metrics | various | — | CPU, memory, event‑loop lag, GC … |
**Source:** `apps/api/src/lib/telemetry.ts`, instrumented in `routes/`.
---
## 3. ML Service (Rust/Axum) — `GET /metrics`
| Endpoint | Description |
|-------------------|--------------------------------------------------|
| `/metrics` | Prometheus text format via `prometheus` crate |
### Metrics
| Metric Name | Type | Labels | Description |
|--------------------------------------------|-----------|--------------------------------|------------------------------------------|
| `zeavis_ml_http_requests_total` | counter | — | Total HTTP requests |
| `zeavis_ml_http_request_duration_seconds` | histogram | — | Request latency buckets |
| `zeavis_ml_http_requests_active` | gauge | — | Concurrently‑handled requests |
| `zeavis_ml_predictions_total` | counter | — | Successful ONNX predictions |
| `zeavis_ml_model_load_status` | gauge | — | 1 = loaded, 0 = not loaded |
| Process metrics (libc/procfs) | various | — | RSS, CPU, fd count … |
**Source:** `apps/ml-service/src/telemetry.rs`, instrumented in `routes.rs`.
---
## Prometheus Auto‑Discovery (Telemetry Stack)
The Telemetry submodule includes a Prometheus instance that uses
`file_sd_configs` to discover targets. Place a target file under
`telemetry/prometheus/targets/` with content such as:
```json
[
{
"targets": ["100.x.x.a:3000"],
"labels": { "service": "zeavis-api", "component": "backend", "env": "production" }
},
{
"targets": ["100.x.x.b:8000"],
"labels": { "service": "zeavis-ml", "component": "inference", "env": "production" }
}
]
```
> ⚠️ **Cross-VPS:** Gunakan **IP Tailscale** (bukan Docker hostname) karena
> Prometheus dan ZeaVis Edu berjalan di VPS berbeda. Pastikan port service
> (`:3000`, `:8000`) terekspos di `0.0.0.0` atau diizinkan oleh aturan
> `iptables`/`ufw` untuk interface Tailscale (`tailscale0`/`100.x.x.x/10`).
The Prometheus config (in `telemetry/prometheus/prometheus.yml`) will
automatically pick up new files within its 15‑second scrape interval —
no restart required.
+151
View File
@@ -0,0 +1,151 @@
# =============================================================================
# ZeaVis Edu — Root Makefile
#
# Orchestrates the application stack (web, api, ml) and the telemetry
# metric pipeline (Prometheus → Ingester → Vector → ClickHouse).
#
# Telemetry commands operate on the submodule at telemetry/.
# =============================================================================
.PHONY: dev build typecheck
.PHONY: telemetry-up telemetry-down telemetry-build telemetry-logs telemetry-restart telemetry-init-db telemetry-test-metric telemetry-status
.PHONY: up-all down-all
SHELL := /bin/bash
# ──────────────────────────────────────────────────────────────────────────────
# Application (Bun / Moon)
# ──────────────────────────────────────────────────────────────────────────────
dev:
bun run dev
build:
bun run build
typecheck:
bun run typecheck
# ──────────────────────────────────────────────────────────────────────────────
# Telemetry Stack
#
# Docker commands reference the telemetry submodule compose file:
# telemetry/deploy/docker-compose.yml
#
# For local development, append the port override:
# make telemetry-up-local
#
# The telmetry compose file is inside the submodule so paths (volumes,
# build context) are relative to telemetry/ — but we run docker compose
# from the project root using -f.
# ──────────────────────────────────────────────────────────────────────────────
TELEMETRY_COMPOSE := telemetry/deploy/docker-compose.yml
TELEMETRY_LOCAL := telemetry/deploy/docker-compose.local.yml
TELEMETRY_ZEAVIS := docker-compose.telemetry.yml
# Start all telemetry services (standalone — cross-VPS production mode)
# Prometheus scrapes ZeaVis Edu via Tailscale IPs, not Docker network.
telemetry-up:
@echo ">> Starting Telemetry stack (standalone)..."
CLICKHOUSE_USER=$${CLICKHOUSE_USER:-telemetry} \
CLICKHOUSE_PASSWORD=$${CLICKHOUSE_PASSWORD:-telemetry} \
docker compose -f $(TELEMETRY_COMPOSE) up -d
@echo ">> Telemetry stack started. Use 'make telemetry-logs' to view output."
# Start telemetry services with ZeaVis Edu network sharing (local single-host dev)
# Prometheus can scrape app services via app-shared-net Docker network.
telemetry-up-local:
@echo ">> Starting Telemetry stack (local dev mode)..."
CLICKHOUSE_USER=$${CLICKHOUSE_USER:-telemetry} \
CLICKHOUSE_PASSWORD=$${CLICKHOUSE_PASSWORD:-telemetry} \
docker compose -f $(TELEMETRY_COMPOSE) -f $(TELEMETRY_LOCAL) -f $(TELEMETRY_ZEAVIS) up -d
@echo ">> Telemetry stack started in local dev mode."
# Stop all telemetry services
telemetry-down:
@echo ">> Stopping Telemetry stack..."
docker compose -f $(TELEMETRY_COMPOSE) -f $(TELEMETRY_ZEAVIS) down
@echo ">> Telemetry stack stopped."
# Build telemetry components (metric-ingester + telemetry-ui)
# Runs inside the telemetry submodule using its own Makefile.
telemetry-build:
@echo ">> Building Telemetry components..."
$(MAKE) -C telemetry build
@echo ">> Telemetry components built."
# Tail telemetry logs (optionally filter by service: s=<name>)
telemetry-logs:
ifdef s
CLICKHOUSE_USER=$${CLICKHOUSE_USER:-telemetry} \
CLICKHOUSE_PASSWORD=$${CLICKHOUSE_PASSWORD:-telemetry} \
docker compose -f $(TELEMETRY_COMPOSE) -f $(TELEMETRY_ZEAVIS) logs -f $(s)
else
CLICKHOUSE_USER=$${CLICKHOUSE_USER:-telemetry} \
CLICKHOUSE_PASSWORD=$${CLICKHOUSE_PASSWORD:-telemetry} \
docker compose -f $(TELEMETRY_COMPOSE) -f $(TELEMETRY_ZEAVIS) logs -f
endif
# Restart a single telemetry service
telemetry-restart:
ifdef s
@echo ">> Restarting service: $(s)..."
CLICKHOUSE_USER=$${CLICKHOUSE_USER:-telemetry} \
CLICKHOUSE_PASSWORD=$${CLICKHOUSE_PASSWORD:-telemetry} \
docker compose -f $(TELEMETRY_COMPOSE) -f $(TELEMETRY_ZEAVIS) restart $(s)
@echo ">> Service $(s) restarted."
else
@echo "Usage: make telemetry-restart s=<service-name>"
@echo "Services: prometheus metric-ingester vector clickhouse query-proxy telemetry-ui"
@exit 1
endif
# Initialize ClickHouse schema
telemetry-init-db:
@echo ">> Initializing ClickHouse schema..."
cd telemetry/clickhouse && DOCKER_CONTAINER=telemetry-clickhouse bash init.sh
@echo ">> Schema initialized."
# Send a test metric through the pipeline
telemetry-test-metric:
@echo ">> Sending test metric to Vector on port 9001..."
curl -X POST http://localhost:9001/metrics \
-H "Content-Type: application/json" \
-d '{"metric_name":"test_zeavis","value":1.0,"timestamp":"$(shell date -u +%Y-%m-%dT%H:%M:%SZ)","labels":{"service":"zeavis-edu"},"env":"dev","region":"local"}'
@echo ""
@echo ">> Metric sent. Check telemetry-logs to verify ingestion."
# Show service status (health check overview)
telemetry-status:
@echo ">> Telemetry stack status:"
@echo ""
@echo "--- Prometheus ---"
-curl -s --max-time 3 http://localhost:9090/-/healthy && echo " healthy" || echo " unhealthy"
@echo ""
@echo "--- Metric Ingester ---"
-curl -s --max-time 3 http://localhost:9091/health || echo " unhealthy"
@echo ""
@echo "--- Vector ---"
-curl -s --max-time 3 http://localhost:9001/health || echo " unhealthy"
@echo ""
@echo "--- ClickHouse ---"
-curl -s --max-time 3 http://localhost:8123/ping && echo " healthy" || echo " unhealthy"
@echo ""
@echo "--- Query Proxy ---"
-curl -s --max-time 3 http://localhost:9092/health || echo " unhealthy"
@echo ""
@echo "--- Telemetry UI ---"
-curl -s --max-time 3 -o /dev/null -w "%{http_code}" http://localhost:8181/ && echo " ok" || echo " unhealthy"
# ──────────────────────────────────────────────────────────────────────────────
# Combined
# ──────────────────────────────────────────────────────────────────────────────
# Start everything (app + telemetry)
up-all: telemetry-up
bun run dev
# Stop everything
down-all: telemetry-down
@echo ">> All services stopped."
+147
View File
@@ -77,6 +77,17 @@ Model klasifikasi menargetkan empat label berbahasa Indonesia:
- GitHub Container Registry
- Traefik labels untuk routing deployment
### Telemetry & Observability
- Prometheus — metric scraping & remote_write
- Metric Ingester (Go) — enrichment, filtering, aggregation
- Vector — buffering & backpressure
- ClickHouse — columnar analytical storage
- Query Proxy (Go) — read-only SQL proxy
- Telemetry UI (Vue 3) — metrics dashboard
- Semua service ZeaVis Edu (web, api, ml-service) mengekspos metrik Prometheus di `/metrics`
- Client-side Web Vitals (CLS, FCP, INP, LCP, TTFB) dikumpulkan di frontend
## Prasyarat
Untuk menjalankan seluruh project secara lokal, siapkan:
@@ -203,6 +214,142 @@ Contoh menjalankan compose setelah environment dan network siap:
docker compose up -d
```
## Telemetry Stack
Proyek ini menyertakan pipeline telemetry metric sebagai git submodule di `telemetry/`. Pipeline mengalirkan metrik dari seluruh service ZeaVis Edu ke ClickHouse untuk analisis dan visualisasi jangka panjang.
### Arsitektur (Production)
Di production, aplikasi dan telemetry berjalan di **VPS terpisah** dan terhubung via **Tailscale** (mesh VPN). Prometheus di VPS telemetry melakukan scrape ke service ZeaVis Edu melalui IP Tailscale masing-masing.
```mermaid
flowchart LR
subgraph VPS1["VPS — ZeaVis Edu (App)"]
W[Web / React<br/>api-zeavisedu.asepharyana.id]
A[API / Elysia<br/>:3000]
M[ML Service / Axum<br/>:8000]
end
subgraph VPS2["VPS — Telemetry Stack"]
P[Prometheus<br/>:9090]
MI[Metric Ingester<br/>:9091]
V[Vector<br/>:9001]
CH[ClickHouse<br/>:8123]
QP[Query Proxy<br/>:9092]
TUI[Telemetry UI<br/>:8181]
end
P -.->|"scrape via Tailscale IP<br/>100.x.x.a:3000/metrics"| A
P -.->|"scrape via Tailscale IP<br/>100.x.x.a:8000/metrics"| M
P -->|remote_write| MI
MI --> V
V --> CH
QP --> CH
TUI --> QP
```
Setiap service ZeaVis Edu mengekspos endpoint `/metrics` dalam format Prometheus text:
| Service | Endpoint | Port (lokal) |
|-----------------------|--------------------|--------------|
| Web (Vite dev) | `GET /metrics` | 5173 |
| API (Elysia) | `GET /metrics` | 3000 |
| ML Service (Axum) | `GET /metrics` | 8000 |
Prometheus di VPS telemetry melakukan **scrape langsung** ke API dan ML service melalui IP Tailscale mereka, bukan melalui domain publik. Konfigurasi target ada di `telemetry/prometheus/targets/zeavis-edu.json` — isi dengan IP Tailscale dari service yang dituju.
Lihat [`METRICS.md`](./METRICS.md) untuk daftar lengkap metrik yang diekspos.
### Service Telemetry
| # | Service | Peran | Port |
|---|---------|------|------|
| 1 | **Prometheus** | Metric scraping & remote_write | 9090 |
| 2 | **Metric Ingester** | Enrichment, filtering, aggregation | 9091 |
| 3 | **Vector** | Buffering, backpressure, retry | 9001 |
| 4 | **ClickHouse** | Columnar analytical storage | 8123 / 9000 |
| 5 | **Query Proxy** | Read-only SQL proxy, tenant isolation | 9092 |
| 6 | **Telemetry UI** | Vue 3 metrics dashboard | 8181 |
### Arsitektur (Local Dev)
Untuk development lokal di satu mesin, telemetry dan app bisa jalan bareng di satu Docker host. Prometheus bisa scrape service lewat Docker network yang sama.
```bash
# Setup network
docker network create app-shared-net
# Build & start telemetry (dengan network sharing)
make telemetry-up-local
```
### Menjalankan Telemetry Stack
Semua operasi telemetry dijalankan dari **root proyek** melalui Makefile:
```bash
# Build komponen telemetry (metric-ingester + telemetry-ui)
make telemetry-build
# Start semua service telemetry (mode produksi, via Tailscale)
make telemetry-up
# Start semua service telemetry (mode lokal — port langsung terbuka)
make telemetry-up-local
# Cek status kesehatan semua service
make telemetry-status
# Lihat log (semua service, atau filter dengan s=)
make telemetry-logs
make telemetry-logs s=metric-ingester
# Restart service tertentu
make telemetry-restart s=prometheus
# Kirim test metric
make telemetry-test-metric
# Stop semua service
make telemetry-down
```
Untuk development lokal:
```bash
# Setup network jika belum ada
docker network create telemetry-net
docker network create app-shared-net
# Build & start
make telemetry-build
make telemetry-up-local
# Buka dashboard di http://localhost:8181
```
### Prometheus Auto-Discovery
Prometheus menggunakan `file_sd_configs` untuk menemukan target secara dinamis. Cukup letakkan file JSON di `telemetry/prometheus/targets/` dan Prometheus akan otomatis mendeteksinya dalam 15 detik — tanpa restart.
File template sudah tersedia di [`telemetry/prometheus/targets/zeavis-edu.json`](telemetry/prometheus/targets/zeavis-edu.json). **Sebelum production, isi `__CHANGE_ME__` dengan IP Tailscale masing-masing service:**
```json
[
{ "targets": ["100.x.x.a:3000"], "labels": { "service": "zeavis-api", "component": "backend", "env": "production" } },
{ "targets": ["100.x.x.a:8000"], "labels": { "service": "zeavis-ml", "component": "inference", "env": "production" } }
]
```
> **Catatan:** Aplikasi ZeaVis Edu mengekspose port Docker-nya (`:3000`, `:8000`) langsung ke host via `docker-compose.yml`. Pastikan port-port tersebut terbuka di network Tailscale (biasanya iptables Tailscale mengizinkan koneksi ke port localhost).
### Environment Variables Telemetry
| Variable | Default | Deskripsi |
|----------|---------|-----------|
| `CLICKHOUSE_USER` | `telemetry` | User ClickHouse |
| `CLICKHOUSE_PASSWORD` | `telemetry` | Password ClickHouse |
## Workflow Machine Learning
Detail lengkap tersedia di [`Machine_Learning/README.md`](Machine_Learning/README.md). Ringkasnya:
+4
View File
@@ -0,0 +1,4 @@
.claude/
.codegraph/
+7 -1
View File
@@ -13,11 +13,17 @@
},
"dependencies": {
"@elysiajs/cors": "1.4.2",
"@opentelemetry/api": "1.9.1",
"@opentelemetry/exporter-prometheus": "0.218.0",
"@opentelemetry/instrumentation-http": "0.218.0",
"@opentelemetry/sdk-node": "0.218.0",
"@opentelemetry/semantic-conventions": "1.41.1",
"@zeavis/shared": "workspace:*",
"bcryptjs": "^3.0.3",
"drizzle-orm": "^0.45.2",
"elysia": "^1.4.28",
"postgres": "^3.4.9"
"postgres": "^3.4.9",
"prom-client": "15.1.3"
},
"devDependencies": {
"@types/bcryptjs": "^3.0.0",
+21
View File
@@ -9,6 +9,9 @@ import { diagnosisRoutes } from './routes/diagnoses';
import { dashboardRoutes } from './routes/dashboard';
import { authRoutes } from './routes/auth';
import { expertRoutes } from './routes/expert';
import { metricsRoutes } from './routes/metrics';
import { httpRequestCounter, httpRequestDuration, httpRequestsActive } from './lib/telemetry';
import './types';
assertRequiredEnv();
@@ -17,6 +20,24 @@ const app = new Elysia()
origin: env.webAppUrl,
credentials: true,
}))
.use(metricsRoutes)
.onBeforeHandle(({ request, path }) => {
httpRequestsActive.inc();
request.metricsStart = performance.now();
request.metricsPath = path;
})
.onAfterHandle(({ request, set }) => {
const start = (request as any).metricsStart as number | undefined;
const path = (request as any).metricsPath as string | undefined;
if (start && path) {
const duration = (performance.now() - start) / 1000;
const method = request.method;
const status = set.status ?? 200;
httpRequestCounter.labels(method, path, String(status)).inc();
httpRequestDuration.labels(method, path).observe(duration);
}
httpRequestsActive.dec();
})
.use(healthRoutes)
.use(statusRoutes)
.use(authRoutes)
+62
View File
@@ -0,0 +1,62 @@
import { Registry, Counter, Histogram, Gauge, collectDefaultMetrics } from 'prom-client';
const registry = new Registry();
// Collect default Node.js metrics (CPU, memory, event loop, etc.)
collectDefaultMetrics({ register: registry });
// ── HTTP Metrics ────────────────────────────────────────
export const httpRequestCounter = new Counter({
name: 'zeavis_api_http_requests_total',
help: 'Total number of HTTP requests handled by the API',
labelNames: ['method', 'path', 'status'] as const,
registers: [registry],
});
export const httpRequestDuration = new Histogram({
name: 'zeavis_api_http_request_duration_seconds',
help: 'Histogram of HTTP request durations in seconds',
labelNames: ['method', 'path'] as const,
buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
registers: [registry],
});
export const httpRequestsActive = new Gauge({
name: 'zeavis_api_http_requests_active',
help: 'Number of HTTP requests currently being processed',
registers: [registry],
});
// ── Business Metrics ────────────────────────────────────
export const classificationCounter = new Counter({
name: 'zeavis_api_classifications_total',
help: 'Total number of classification predictions requested via API',
labelNames: ['result'] as const,
registers: [registry],
});
export const diagnosisCounter = new Counter({
name: 'zeavis_api_diagnoses_total',
help: 'Total number of diagnoses created',
labelNames: ['disease'] as const,
registers: [registry],
});
export const authCounter = new Counter({
name: 'zeavis_api_auth_operations_total',
help: 'Total authentication operations (login, register, refresh)',
labelNames: ['operation', 'success'] as const,
registers: [registry],
});
// ── Export ──────────────────────────────────────────────
export function getMetricsContentType(): string {
return registry.contentType;
}
export async function getMetrics(): Promise<string> {
return await registry.metrics();
}
+6
View File
@@ -16,6 +16,7 @@ import {
verifyPassword,
} from '../lib/auth';
import { env } from '../config/env';
import { authCounter } from '../lib/telemetry';
function normalizeEmail(email: unknown) {
return typeof email === 'string' ? email.trim().toLowerCase() : '';
@@ -63,6 +64,8 @@ export const authRoutes = new Elysia({ prefix: '/api/v1/auth' })
const token = await createSession(user.id);
set.headers['Set-Cookie'] = createSessionCookie(token);
authCounter.labels('register', 'true').inc();
return {
user: {
id: user.id,
@@ -90,12 +93,15 @@ export const authRoutes = new Elysia({ prefix: '/api/v1/auth' })
const user = rows[0];
if (!user?.passwordHash || !(await verifyPassword(req!.password!, user.passwordHash))) {
authCounter.labels('login', 'false').inc();
return badRequest('Invalid email or password');
}
const token = await createSession(user.id);
set.headers['Set-Cookie'] = createSessionCookie(token);
authCounter.labels('login', 'true').inc();
return {
user: {
id: user.id,
+3
View File
@@ -16,6 +16,7 @@ import { desc, eq } from 'drizzle-orm';
import { classifyImage } from '../lib/image-model';
import { uploadImageToStorage } from '../lib/uploader-client';
import { toDisease } from '../lib/disease-mappers';
import { classificationCounter } from '../lib/telemetry';
function toImageClassificationRecord(row: {
id: string;
@@ -141,6 +142,8 @@ export const classificationRoutes = new Elysia({ prefix: '/api/v1' })
);
}
classificationCounter.labels(classificationResult.predictedDiseaseSlug).inc();
const db = createDbClient();
let diseaseRow;
try {
+7
View File
@@ -9,6 +9,7 @@ import { getCurrentUser } from '../lib/auth';
import { classifyImage } from '../lib/image-model';
import { uploadImageToStorage } from '../lib/uploader-client';
import { env } from '../config/env';
import { diagnosisCounter } from '../lib/telemetry';
interface ReviewRow {
reviewId: string | null;
@@ -222,6 +223,9 @@ export const diagnosisRoutes = new Elysia({ prefix: '/api/v1' })
const record = await loadDiagnosisRecord(inserted[0].id, user.id, false);
if (!record) return serviceUnavailable('Database unavailable');
diagnosisCounter.labels(record.predictedDiseaseSlug ?? 'unknown').inc();
return record;
} catch (error) {
const inserted = await db
@@ -240,6 +244,9 @@ export const diagnosisRoutes = new Elysia({ prefix: '/api/v1' })
const record = await loadDiagnosisRecord(inserted[0].id, user.id, false);
if (!record) return serviceUnavailable('Database unavailable');
diagnosisCounter.labels('failed').inc();
return record;
}
})
+10
View File
@@ -0,0 +1,10 @@
import { Elysia } from 'elysia';
import { getMetrics, getMetricsContentType } from '../lib/telemetry';
export const metricsRoutes = new Elysia()
.get('/metrics', async () => {
const body = await getMetrics();
return new Response(body, {
headers: { 'Content-Type': getMetricsContentType() },
});
});
+8
View File
@@ -0,0 +1,8 @@
declare global {
interface Request {
metricsStart?: number;
metricsPath?: string;
}
}
export {};
+4 -1
View File
@@ -1,3 +1,6 @@
.venv/
__pycache__/
target/
target/
.claude/
.codegraph/
+197 -13
View File
@@ -111,7 +111,7 @@ dependencies = [
"num-traits",
"pastey",
"rayon",
"thiserror",
"thiserror 2.0.18",
"v_frame",
"y4m",
]
@@ -402,7 +402,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys",
"windows-sys 0.61.2",
]
[[package]]
@@ -457,6 +457,12 @@ dependencies = [
"miniz_oxide",
]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
@@ -587,6 +593,12 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hmac-sha256"
version = "1.1.14"
@@ -801,6 +813,12 @@ dependencies = [
"cc",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
@@ -902,7 +920,7 @@ checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
dependencies = [
"libc",
"wasi",
"windows-sys",
"windows-sys 0.61.2",
]
[[package]]
@@ -1000,7 +1018,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys",
"windows-sys 0.61.2",
]
[[package]]
@@ -1253,6 +1271,28 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "procfs"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f"
dependencies = [
"bitflags",
"hex",
"procfs-core",
"rustix 0.38.44",
]
[[package]]
name = "procfs-core"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec"
dependencies = [
"bitflags",
"hex",
]
[[package]]
name = "profiling"
version = "1.0.18"
@@ -1272,6 +1312,43 @@ dependencies = [
"syn",
]
[[package]]
name = "prometheus"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
dependencies = [
"cfg-if",
"fnv",
"lazy_static",
"libc",
"memchr",
"parking_lot",
"procfs",
"protobuf",
"thiserror 2.0.18",
]
[[package]]
name = "protobuf"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror 1.0.69",
]
[[package]]
name = "protobuf-support"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "pxfm"
version = "0.1.29"
@@ -1373,7 +1450,7 @@ dependencies = [
"rand",
"rand_chacha",
"simd_helpers",
"thiserror",
"thiserror 2.0.18",
"v_frame",
"wasm-bindgen",
]
@@ -1451,6 +1528,19 @@ version = "0.8.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4"
[[package]]
name = "rustix"
version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys 0.4.15",
"windows-sys 0.59.0",
]
[[package]]
name = "rustix"
version = "1.1.4"
@@ -1460,8 +1550,8 @@ dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
"linux-raw-sys 0.12.1",
"windows-sys 0.61.2",
]
[[package]]
@@ -1491,7 +1581,7 @@ version = "0.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
dependencies = [
"windows-sys",
"windows-sys 0.61.2",
]
[[package]]
@@ -1644,7 +1734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys",
"windows-sys 0.61.2",
]
[[package]]
@@ -1705,8 +1795,17 @@ dependencies = [
"fastrand",
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys",
"rustix 1.1.4",
"windows-sys 0.61.2",
]
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl 1.0.69",
]
[[package]]
@@ -1715,7 +1814,18 @@ version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
dependencies = [
"thiserror-impl",
"thiserror-impl 2.0.18",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -1763,7 +1873,7 @@ dependencies = [
"pin-project-lite",
"socket2",
"tokio-macros",
"windows-sys",
"windows-sys 0.61.2",
]
[[package]]
@@ -2090,6 +2200,15 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-sys"
version = "0.61.2"
@@ -2099,6 +2218,70 @@ dependencies = [
"windows-link",
]
[[package]]
name = "windows-targets"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "wit-bindgen"
version = "0.51.0"
@@ -2208,6 +2391,7 @@ dependencies = [
"image",
"ndarray",
"ort",
"prometheus",
"serde",
"serde_json",
"temp-env",
+1
View File
@@ -9,6 +9,7 @@ axum = { version = "0.7", features = ["multipart"] }
image = "0.25"
ndarray = "0.17"
ort = { version = "2.0.0-rc.10", features = ["download-binaries", "ndarray"] }
prometheus = { version = "0.14.0", features = ["process"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "net"] }
+2 -4
View File
@@ -5,7 +5,7 @@ COPY apps/ml-service/Cargo.toml apps/ml-service/Cargo.lock ./
COPY apps/ml-service/src ./src
RUN cargo build --locked --release
FROM debian:trixie-slim AS runner
FROM archlinux:latest AS runner
WORKDIR /app
ENV MODEL_PATH=/app/model/model.onnx
@@ -14,9 +14,7 @@ ENV ML_SERVICE_HOST=0.0.0.0
ENV ML_SERVICE_PORT=8000
ENV RUST_LOG=info
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates \
&& rm -rf /var/lib/apt/lists/*
RUN pacman -Syu --noconfirm ca-certificates 2>/dev/null
COPY --from=builder /app/target/release/zeavis-ml-service /usr/local/bin/zeavis-ml-service
COPY Machine_Learning/model/model.onnx /app/model/model.onnx
+4
View File
@@ -3,6 +3,7 @@ mod error;
mod image;
mod model;
mod routes;
mod telemetry;
use anyhow::Result;
use config::Config;
@@ -32,6 +33,9 @@ async fn main() -> Result<()> {
"Model service initialized"
);
// Set model load status metric
telemetry::model_load_status().set(if model.is_loaded() { 1.0 } else { 0.0 });
// Create AppState
let state = AppState { model };
+22 -3
View File
@@ -3,6 +3,8 @@ use crate::config::{LABELS, SERVICE_NAME, SERVICE_VERSION};
use crate::model::{ModelService, Prediction};
use crate::error::ServiceError;
use crate::image::preprocess_image;
use crate::telemetry;
use crate::telemetry::RequestMetricsGuard;
use axum::{
extract::{State, Multipart},
routing::{get, post},
@@ -64,22 +66,34 @@ pub fn prediction_response(prediction: Prediction) -> PredictionResponse {
}
}
pub async fn metrics() -> (axum::http::StatusCode, String) {
(axum::http::StatusCode::OK, telemetry::encode_metrics())
}
pub async fn health(State(state): State<AppState>) -> Json<HealthResponse> {
Json(health_response(state.model.is_loaded()))
let _guard = RequestMetricsGuard::new();
let res = health_response(state.model.is_loaded());
_guard.finish();
Json(res)
}
pub async fn metadata(State(state): State<AppState>) -> Json<MetadataResponse> {
Json(metadata_response(
let _guard = RequestMetricsGuard::new();
let res = metadata_response(
state.model.model_path().to_string_lossy().to_string(),
state.model.is_loaded(),
state.model.input_size(),
))
);
_guard.finish();
Json(res)
}
pub async fn predict(
State(state): State<AppState>,
mut multipart: Multipart,
) -> Result<Json<PredictionResponse>, ServiceError> {
let _guard = RequestMetricsGuard::new();
// Extract the file field from multipart
let mut file_data = None;
while let Ok(Some(field)) = multipart.next_field().await {
@@ -121,6 +135,10 @@ pub async fn predict(
// Run prediction
let prediction = state.model.predict(input)?;
// Record business and request telemetry
telemetry::predictions_total().inc();
_guard.finish();
Ok(Json(prediction_response(prediction)))
}
@@ -129,6 +147,7 @@ pub fn router(state: AppState) -> Router {
.route("/health", get(health))
.route("/metadata", get(metadata))
.route("/predict", post(predict))
.route("/metrics", get(metrics))
.with_state(state)
}
+118
View File
@@ -0,0 +1,118 @@
use prometheus::{Counter, Gauge, Histogram, HistogramOpts, Registry, TextEncoder};
use std::sync::OnceLock;
use std::time::Instant;
fn global_registry() -> &'static Registry {
static REGISTRY: OnceLock<Registry> = OnceLock::new();
REGISTRY.get_or_init(|| {
Registry::new_custom(Some("zeavis_ml".to_string()), None).expect("create registry")
})
}
macro_rules! define_metric {
($name:ident, $ty:ty, $init:expr) => {
pub fn $name() -> &'static $ty {
static METRIC: OnceLock<$ty> = OnceLock::new();
METRIC.get_or_init(|| {
let m = $init;
global_registry()
.register(Box::new(m.clone()))
.expect(concat!("register ", stringify!($name)));
m
})
}
};
}
// ── HTTP Metrics ────────────────────────────────────────
define_metric!(
http_requests_total,
Counter,
Counter::new("zeavis_ml_http_requests_total", "Total number of HTTP requests")
.expect("create counter")
);
define_metric!(
http_request_duration_seconds,
Histogram,
Histogram::with_opts(
HistogramOpts::new(
"zeavis_ml_http_request_duration_seconds",
"HTTP request duration in seconds",
)
.buckets(vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]),
)
.expect("create histogram")
);
define_metric!(
http_requests_active,
Gauge,
Gauge::new(
"zeavis_ml_http_requests_active",
"Number of active HTTP requests",
)
.expect("create gauge")
);
// ── Business Metrics ────────────────────────────────────
define_metric!(
predictions_total,
Counter,
Counter::new(
"zeavis_ml_predictions_total",
"Total number of prediction requests",
)
.expect("create counter")
);
define_metric!(
model_load_status,
Gauge,
Gauge::new(
"zeavis_ml_model_load_status",
"Model load status (1 = loaded, 0 = not loaded)",
)
.expect("create gauge")
);
// ── Request Guard (Drop-based cleanup for active gauge) ─
pub struct RequestMetricsGuard {
start: Instant,
}
impl RequestMetricsGuard {
pub fn new() -> Self {
http_requests_active().inc();
Self {
start: Instant::now(),
}
}
/// Record duration and request count before the guard drops.
pub fn finish(&self) {
http_request_duration_seconds().observe(self.start.elapsed().as_secs_f64());
http_requests_total().inc();
}
}
impl Drop for RequestMetricsGuard {
fn drop(&mut self) {
http_requests_active().dec();
}
}
// ── Export ──────────────────────────────────────────────
pub fn encode_metrics() -> String {
let encoder = TextEncoder::new();
let mut buffer = String::new();
let metric_families = global_registry().gather();
encoder
.encode_utf8(&metric_families, &mut buffer)
.unwrap();
buffer
}
+4
View File
@@ -0,0 +1,4 @@
.claude/
.codegraph/
+10 -3
View File
@@ -14,8 +14,15 @@ ENV VITE_API_BASE_URL=$VITE_API_BASE_URL
RUN bun run --cwd packages/shared build
RUN bun run --cwd apps/web build
FROM nginx:1.27-alpine AS runner
COPY apps/web/nginx.conf /etc/nginx/conf.d/default.conf
COPY --from=builder /app/apps/web/dist /usr/share/nginx/html
# =============================================================================
# Stage 2: nginx (Alpine package — includes sub_filter module)
# Alpine's nginx package is built with --with-http_sub_module
# =============================================================================
FROM alpine:3.20 AS runner
RUN apk add --no-cache nginx
COPY apps/web/nginx.conf /etc/nginx/http.d/default.conf
COPY --from=builder /app/apps/web/dist /var/lib/nginx/html
EXPOSE 80
CMD ["nginx", "-g", "daemon off;"]
+10 -1
View File
@@ -1,7 +1,7 @@
server {
listen 80;
server_name _;
root /usr/share/nginx/html;
root /var/lib/nginx/html;
index index.html;
location /api/ {
@@ -12,6 +12,15 @@ server {
proxy_set_header X-Forwarded-Proto $scheme;
}
# Expose API metrics through the web endpoint (Prometheus scrape target)
location /metrics {
proxy_pass http://zeavis-api:3000/metrics;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
location / {
try_files $uri $uri/ /index.html;
}
+2
View File
@@ -20,7 +20,9 @@
"react": "^19.2.6",
"react-dom": "^19.2.6",
"react-router-dom": "^7.15.1",
"recharts": "3.8.1",
"tailwind-merge": "^3.6.0",
"web-vitals": "5.3.0",
"zustand": "^5.0.13"
},
"devDependencies": {
+19
View File
@@ -14,6 +14,7 @@ import { DiseaseDetailPage } from "@/pages/disease-detail-page";
import { DiagnosisDetailPage } from "@/pages/diagnosis-detail-page";
import { ExpertReviewsPage } from "@/pages/expert-reviews-page";
import { DiagnosesPage } from "@/pages/diagnoses-page";
import { TelemetryPage } from "@/pages/telemetry-page";
import { LoginPage } from "@/pages/login-page";
import { RegisterPage } from "@/pages/register-page";
import { MainLayout } from "@/components/layout/main-layout";
@@ -36,6 +37,7 @@ function LogoutProses() {
return <Navigate to="/login" replace />;
}
import { trackPageView } from "./lib/telemetry";
const queryClient = new QueryClient();
@@ -129,12 +131,29 @@ const router = createBrowserRouter([
<LogoutProses />
),
},
{
path: "/telemetry",
element: (
<MainLayout>
<TelemetryPage />
</MainLayout>
),
},
]);
function PageViewTracker() {
const location = window.location;
useEffect(() => {
trackPageView(location.pathname + location.search);
}, [location.pathname, location.search]);
return null;
}
export function App() {
return (
<QueryClientProvider client={queryClient}>
<AuthInitializer />
<PageViewTracker />
<RouterProvider router={router} />
</QueryClientProvider>
);
@@ -13,6 +13,7 @@ const NAV_ITEMS = [
{ path: "/diagnoses", label: "Diagnosa" },
{ path: "/catalog", label: "Pustaka", altPath: "/library" },
{ path: "/expert/reviews", label: "Review" },
{ path: "/telemetry", label: "Telemetry" },
];
export function MobileNav({ open, onClose }: Props) {
@@ -71,6 +71,12 @@ export function Navbar() {
>
Keluar <LogOut className="ml-2 h-4 w-4" />
</Link>
<Link
to="/telemetry"
className={navLinkClassName(isActive("/telemetry"))}
>
Telemetry
</Link>
</nav>
<button
+52
View File
@@ -0,0 +1,52 @@
/**
* Client‑side telemetry for the ZeaVis Edu web app.
*
* In development, metrics are collected in‑memory and exposed at /metrics
* via a Vite plugin. In production they are sent as HTTP beacons to the
* Telemetry pipeline (see METRICS.md).
*/
// ── Web Vitals ──────────────────────────────────────────
export type MetricEntry = {
name: string;
value: number;
rating?: string;
};
const vitalsBuffer: MetricEntry[] = [];
export function reportWebVitals(metric: MetricEntry): void {
vitalsBuffer.push(metric);
// Keep last 20 entries in memory for the /metrics endpoint
if (vitalsBuffer.length > 20) vitalsBuffer.shift();
console.debug(`[telemetry] ${metric.name}: ${metric.value} (${metric.rating ?? 'n/a'})`);
}
// ── Page‑view counter ───────────────────────────────────
let pageViewCount = 0;
export function trackPageView(path: string): void {
pageViewCount++;
console.debug(`[telemetry] pageview: ${path} (total: ${pageViewCount})`);
}
// ── Metrics serialisation (consumed by vite‑plugin) ────
export function collectMetrics(): string {
const lines: string[] = [];
// ── Default process‑like metrics ──────────────────────
lines.push('# HELP zeavis_web_page_views_total Total page views');
lines.push('# TYPE zeavis_web_page_views_total counter');
lines.push(`zeavis_web_page_views_total ${pageViewCount}`);
lines.push('# HELP zeavis_web_vital_bucket Web Vitals observed this session');
lines.push('# TYPE zeavis_web_vital_bucket gauge');
for (const v of vitalsBuffer) {
lines.push(`zeavis_web_vital_bucket{name="${v.name}",rating="${v.rating ?? 'unknown'}"} ${v.value}`);
}
return lines.join('\n') + '\n';
}
+9
View File
@@ -2,6 +2,15 @@ import React from "react";
import ReactDOM from "react-dom/client";
import { App } from "./app";
import "./index.css";
import { reportWebVitals } from "./lib/telemetry";
import { onCLS, onFCP, onINP, onLCP, onTTFB } from "web-vitals";
// Report Web Vitals to our in-memory telemetry store
onCLS((m) => reportWebVitals({ name: "CLS", value: m.value, rating: m.rating }));
onFCP((m) => reportWebVitals({ name: "FCP", value: m.value, rating: m.rating }));
onINP((m) => reportWebVitals({ name: "INP", value: m.value, rating: m.rating }));
onLCP((m) => reportWebVitals({ name: "LCP", value: m.value, rating: m.rating }));
onTTFB((m) => reportWebVitals({ name: "TTFB", value: m.value, rating: m.rating }));
ReactDOM.createRoot(document.getElementById("root")!).render(
<React.StrictMode>
+570
View File
@@ -0,0 +1,570 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import {
BarChart3,
Activity,
Cpu,
HardDrive,
Database,
Layers,
RefreshCw,
AlertTriangle,
} from "lucide-react";
import {
AreaChart,
Area,
XAxis,
YAxis,
CartesianGrid,
Tooltip,
ResponsiveContainer,
BarChart,
Bar,
} from "recharts";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
// ─── Types ──────────────────────────────────────────────────────────
const API_BASE = "https://telemetry.imrnes.team/proxy/dashboard";
interface DashboardStats {
cpu_usage: number;
disk_usage: number;
total_metrics: number;
active_services: number;
uptime_seconds: number;
health: { disk_readonly: boolean; errors: number };
}
interface DiscoveredMetric {
metric_name: string;
service: string;
sample_count: number;
latest_value: number;
}
interface ChartPoint {
time: string;
value: number;
}
// ─── Helpers ─────────────────────────────────────────────────────────
function fmt(n: number): string {
if (n >= 1_000_000) return (n / 1_000_000).toFixed(1) + "M";
if (n >= 1_000) return (n / 1_000).toFixed(1) + "K";
return n.toFixed(1);
}
function fmtDuration(s: number): string {
const d = Math.floor(s / 86400);
const h = Math.floor((s % 86400) / 3600);
if (d > 0) return `${d}d ${h}h`;
const m = Math.floor((s % 3600) / 60);
if (h > 0) return `${h}h ${m}m`;
return `${m}m`;
}
function fmtPct(v: number): string {
return (v * 100).toFixed(1) + "%";
}
// ─── API Client ──────────────────────────────────────────────────────
class TelemetryAPI {
private base: string;
constructor(base: string) {
this.base = base;
}
async stats(): Promise<DashboardStats> {
const res = await fetch(`${this.base}/stats`);
if (!res.ok) throw new Error(`Stats API: ${res.status}`);
return res.json();
}
async discover(): Promise<DiscoveredMetric[]> {
const res = await fetch(`${this.base}/discover`);
if (!res.ok) throw new Error(`Discover API: ${res.status}`);
const data = await res.json();
return data.metrics ?? [];
}
async charts(
panels: { key: string; metric: string; aggregation?: string }[],
): Promise<Map<string, ChartPoint[]>> {
const now = Math.floor(Date.now() / 1000);
const oneHourAgo = now - 3600;
const res = await fetch(`${this.base}/charts`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
panels: panels.map((p) => ({
key: p.key,
metric: p.metric,
start: oneHourAgo,
end: now,
aggregation: p.aggregation ?? "avg",
})),
}),
});
if (!res.ok) throw new Error(`Charts API: ${res.status}`);
const data = await res.json();
const map = new Map<string, ChartPoint[]>();
for (const r of data.results ?? []) {
map.set(r.key, r.data ?? []);
}
return map;
}
}
const api = new TelemetryAPI(API_BASE);
// ─── Stat Card ───────────────────────────────────────────────────────
function StatCard({
icon: Icon,
label,
value,
sub,
color,
}: {
icon: typeof BarChart3;
label: string;
value: string;
sub?: string;
color: string;
}) {
return (
<Card className="border-slate-200 shadow-sm">
<CardHeader className="flex flex-row items-center justify-between pb-2 pt-4 px-4">
<CardTitle className="text-sm font-medium text-slate-500">
{label}
</CardTitle>
<Icon className={`h-4 w-4 ${color}`} />
</CardHeader>
<CardContent className="px-4 pb-4">
<div className="text-2xl font-bold">{value}</div>
{sub && <p className="text-xs text-slate-400 mt-1">{sub}</p>}
</CardContent>
</Card>
);
}
// ─── Metric Card ─────────────────────────────────────────────────────
function MetricChart({
title,
data,
loading,
color,
}: {
title: string;
data: ChartPoint[];
loading: boolean;
color: string;
}) {
if (loading) {
return (
<Card className="border-slate-200 shadow-sm">
<CardHeader className="pb-2 px-4 pt-4">
<CardTitle className="text-sm font-medium">{title}</CardTitle>
</CardHeader>
<CardContent className="px-4 pb-4">
<div className="flex items-center justify-center h-40 text-slate-400 text-sm">
Loading...
</div>
</CardContent>
</Card>
);
}
if (!data || data.length === 0) {
return (
<Card className="border-slate-200 shadow-sm">
<CardHeader className="pb-2 px-4 pt-4">
<CardTitle className="text-sm font-medium">{title}</CardTitle>
</CardHeader>
<CardContent className="px-4 pb-4">
<div className="flex items-center justify-center h-40 text-slate-400 text-sm">
No data available
</div>
</CardContent>
</Card>
);
}
return (
<Card className="border-slate-200 shadow-sm">
<CardHeader className="pb-2 px-4 pt-4">
<CardTitle className="text-sm font-medium">{title}</CardTitle>
</CardHeader>
<CardContent className="px-4 pb-4">
<ResponsiveContainer width="100%" height={160}>
<AreaChart data={data}>
<defs>
<linearGradient id={`grad-${title}`} x1="0" y1="0" x2="0" y2="1">
<stop offset="5%" stopColor={color} stopOpacity={0.2} />
<stop offset="95%" stopColor={color} stopOpacity={0} />
</linearGradient>
</defs>
<CartesianGrid strokeDasharray="3 3" stroke="#e2e8f0" />
<XAxis
dataKey="time"
tick={{ fontSize: 10 }}
tickFormatter={(v) => {
const d = new Date(v);
return `${d.getHours().toString().padStart(2, "0")}:${d.getMinutes().toString().padStart(2, "0")}`;
}}
/>
<YAxis tick={{ fontSize: 10 }} />
<Tooltip
labelFormatter={(v) => new Date(v).toLocaleTimeString()}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
formatter={(val: any) => [typeof val === "number" ? val.toFixed(2) : String(val ?? ""), title]}
/>
<Area
type="monotone"
dataKey="value"
stroke={color}
fill={`url(#grad-${title})`}
strokeWidth={2}
/>
</AreaChart>
</ResponsiveContainer>
</CardContent>
</Card>
);
}
// ─── Main Page ───────────────────────────────────────────────────────
export function TelemetryPage() {
const [stats, setStats] = useState<DashboardStats | null>(null);
const [metrics, setMetrics] = useState<DiscoveredMetric[]>([]);
const [chartMap, setChartMap] = useState<Map<string, ChartPoint[]>>(new Map());
const [memChartData, setMemChartData] = useState<ChartPoint[] | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [refreshing, setRefreshing] = useState(false);
const intervalRef = useRef<number | undefined>(undefined);
const fetchData = useCallback(async (isRefresh = false) => {
try {
if (isRefresh) setRefreshing(true);
else setLoading(true);
setError(null);
const [statsData, discoverData] = await Promise.all([
api.stats(),
api.discover(),
]);
setStats(statsData);
setMetrics(discoverData);
// Fetch charts for top metrics (excluding prometheus noise)
const topMetrics = discoverData
.filter((m) => !m.metric_name.startsWith("prometheus_") && m.service !== "prometheus")
.slice(0, 4);
const chartPanels = topMetrics.map((m) => ({
key: m.metric_name,
metric: m.metric_name,
}));
// Built-in computations + app metrics
chartPanels.unshift({ key: "_cpu_usage_pct", metric: "_cpu_usage_pct" });
chartPanels.unshift({ key: "_disk_usage_pct", metric: "_disk_usage_pct" });
// Memory charts (raw values, % computed client-side)
chartPanels.push({ key: "node_memory_MemTotal_bytes", metric: "node_memory_MemTotal_bytes" });
chartPanels.push({ key: "node_memory_MemAvailable_bytes", metric: "node_memory_MemAvailable_bytes" });
// App-specific metrics
for (const appMetric of ["zeavis_api_http_requests_total", "zeavis_api_http_requests_active"]) {
if (discoverData.find((m) => m.metric_name === appMetric)) {
chartPanels.push({ key: appMetric, metric: appMetric });
}
}
const charts = await api.charts(chartPanels);
setChartMap(charts);
// Compute memory usage % = (total - available) / total
const memTotalData = charts.get("node_memory_MemTotal_bytes");
const memAvailData = charts.get("node_memory_MemAvailable_bytes");
if (memTotalData && memAvailData && memTotalData.length > 0 && memAvailData.length > 0) {
const merged: ChartPoint[] = [];
for (let i = 0; i < Math.min(memTotalData.length, memAvailData.length); i++) {
const total = memTotalData[i].value;
const avail = memAvailData[i].value;
if (total > 0) {
merged.push({ time: memTotalData[i].time, value: 1 - avail / total });
}
}
setMemChartData(merged);
} else {
setMemChartData([]);
}
setChartMap(charts);
} catch (e) {
setError(e instanceof Error ? e.message : "Unknown error");
} finally {
setLoading(false);
setRefreshing(false);
}
}, []);
useEffect(() => {
fetchData();
intervalRef.current = window.setInterval(() => fetchData(true), 30_000);
return () => {
if (intervalRef.current) clearInterval(intervalRef.current);
};
}, [fetchData]);
const serviceCounts = useMemo(() => {
const counts = new Map<string, number>();
for (const m of metrics) {
counts.set(m.service, (counts.get(m.service) ?? 0) + 1);
}
return Array.from(counts.entries()).sort((a, b) => b[1] - a[1]);
}, [metrics]);
const topMetrics = useMemo(() => {
return metrics.slice(0, 10);
}, [metrics]);
if (loading && !stats) {
return (
<div className="space-y-6">
<div className="flex items-center justify-center h-64">
<div className="text-center space-y-3">
<div className="inline-block h-8 w-8 animate-spin rounded-full border-4 border-solid border-[#48A111] border-r-transparent" />
<p className="text-sm text-muted-foreground">
Loading telemetry data...
</p>
</div>
</div>
</div>
);
}
if (error && !stats) {
return (
<div className="space-y-6">
<div className="flex items-center justify-center h-64">
<div className="text-center space-y-3 max-w-md">
<AlertTriangle className="h-10 w-10 text-red-500 mx-auto" />
<p className="text-sm font-semibold text-red-600">
Failed to load telemetry data
</p>
<p className="text-xs text-slate-500">{error}</p>
<button
onClick={() => fetchData()}
className="inline-flex items-center gap-1 rounded-full bg-[#48A111] px-4 py-2 text-sm font-semibold text-white hover:bg-[#306D29]"
>
Retry
</button>
</div>
</div>
</div>
);
}
return (
<div className="space-y-6">
{/* Header */}
<div className="flex items-center justify-between">
<div>
<h1 className="text-2xl md:text-[28px] font-extrabold text-[#214B11] flex items-center gap-3">
<BarChart3 className="h-7 w-7 text-[#48A111]" />
Telemetry Dashboard
</h1>
<p className="text-sm text-muted-foreground mt-1">
System metrics from Prometheus pipeline via ClickHouse.
{error && (
<span className="text-amber-600 ml-2">
(partial data — {error})
</span>
)}
</p>
</div>
<button
onClick={() => fetchData(true)}
disabled={refreshing}
className="inline-flex items-center gap-1 rounded-full border border-slate-300 bg-white px-3 py-1.5 text-sm font-medium text-slate-600 hover:bg-slate-50 transition-colors disabled:opacity-50"
>
<RefreshCw
className={`h-4 w-4 ${refreshing ? "animate-spin" : ""}`}
/>
Refresh
</button>
</div>
{/* Stat Cards */}
{stats && (
<div className="grid gap-4 md:grid-cols-2 lg:grid-cols-4">
<StatCard
icon={Cpu}
label="CPU Usage"
value={fmtPct(stats.cpu_usage)}
color="text-blue-600"
/>
<StatCard
icon={HardDrive}
label="Disk Usage"
value={fmtPct(stats.disk_usage)}
color="text-amber-600"
/>
<StatCard
icon={Database}
label="Total Metrics"
value={fmt(stats.total_metrics)}
sub={`${stats.active_services} active services`}
color="text-green-600"
/>
<StatCard
icon={Activity}
label="Uptime"
value={fmtDuration(stats.uptime_seconds)}
sub={
stats.health.errors > 0
? `${stats.health.errors} errors`
: "All healthy"
}
color={stats.health.errors > 0 ? "text-red-600" : "text-green-600"}
/>
</div>
)}
{/* CPU, Memory, Disk charts */}
<div className="grid gap-4 md:grid-cols-2 lg:grid-cols-3">
<MetricChart
title="CPU Usage (last hour)"
data={chartMap.get("_cpu_usage_pct") ?? []}
loading={loading}
color="#2563eb"
/>
<MetricChart
title="Memory Usage (last hour)"
data={memChartData ?? []}
loading={loading}
color="#8b5cf6"
/>
<MetricChart
title="Disk Usage (last hour)"
data={chartMap.get("_disk_usage_pct") ?? []}
loading={loading}
color="#f59e0b"
/>
</div>
{/* Application Metrics */}
{(chartMap.has("zeavis_api_http_requests_total") || chartMap.has("zeavis_api_http_requests_active")) && (
<section className="space-y-4">
<h3 className="text-lg font-semibold text-[#214B11]">Application Metrics</h3>
<div className="grid gap-4 md:grid-cols-1 lg:grid-cols-2">
{chartMap.has("zeavis_api_http_requests_total") && (
<MetricChart
title="HTTP Requests Total (API)"
data={chartMap.get("zeavis_api_http_requests_total") ?? []}
loading={loading}
color="#ec4899"
/>
)}
{chartMap.has("zeavis_api_http_requests_active") && (
<MetricChart
title="Active HTTP Requests (API)"
data={chartMap.get("zeavis_api_http_requests_active") ?? []}
loading={loading}
color="#14b8a6"
/>
)}
</div>
</section>
)}
{/* Top Metrics */}
<Card className="border-slate-200 shadow-sm">
<CardHeader className="pb-2 px-4 pt-4">
<CardTitle className="text-lg font-semibold text-[#214B11] flex items-center gap-2">
<Layers className="h-5 w-5 text-[#48A111]" />
Top Metrics
</CardTitle>
</CardHeader>
<CardContent className="px-4 pb-4">
{topMetrics.length === 0 ? (
<div className="text-center py-8 text-slate-400 text-sm">
No metrics discovered yet
</div>
) : (
<div className="overflow-x-auto">
<table className="w-full text-sm">
<thead>
<tr className="border-b border-slate-200">
<th className="text-left py-2 px-2 font-medium text-slate-500">
Metric
</th>
<th className="text-left py-2 px-2 font-medium text-slate-500">
Service
</th>
<th className="text-right py-2 px-2 font-medium text-slate-500">
Samples
</th>
<th className="text-right py-2 px-2 font-medium text-slate-500">
Latest Value
</th>
</tr>
</thead>
<tbody>
{topMetrics.map((m) => (
<tr key={m.metric_name} className="border-b border-slate-100 hover:bg-slate-50">
<td className="py-2 px-2 font-mono text-xs text-slate-700 max-w-[300px] truncate">
{m.metric_name}
</td>
<td className="py-2 px-2">
<span className="inline-flex items-center rounded-full bg-[#EFF6E8] px-2 py-0.5 text-xs font-medium text-[#48A111]">
{m.service}
</span>
</td>
<td className="py-2 px-2 text-right text-slate-600">
{fmt(m.sample_count)}
</td>
<td className="py-2 px-2 text-right font-mono text-xs text-slate-600">
{typeof m.latest_value === "number"
? m.latest_value.toFixed(4)
: String(m.latest_value)}
</td>
</tr>
))}
</tbody>
</table>
</div>
)}
</CardContent>
</Card>
{/* Service Distribution */}
<Card className="border-slate-200 shadow-sm">
<CardHeader className="pb-2 px-4 pt-4">
<CardTitle className="text-lg font-semibold text-[#214B11] flex items-center gap-2">
<Layers className="h-5 w-5 text-[#48A111]" />
Services
</CardTitle>
</CardHeader>
<CardContent className="px-4 pb-4">
{serviceCounts.length === 0 ? (
<div className="text-center py-8 text-slate-400 text-sm">
No services discovered
</div>
) : (
<ResponsiveContainer width="100%" height={200}>
<BarChart data={serviceCounts.map(([name, count]) => ({ name, count }))}>
<CartesianGrid strokeDasharray="3 3" stroke="#e2e8f0" />
<XAxis dataKey="name" tick={{ fontSize: 12 }} />
<YAxis tick={{ fontSize: 12 }} />
<Tooltip />
<Bar dataKey="count" fill="#48A111" radius={[4, 4, 0, 0]} />
</BarChart>
</ResponsiveContainer>
)}
</CardContent>
</Card>
</div>
);
}
+1 -1
View File
@@ -7,5 +7,5 @@
"moduleResolution": "Bundler",
"types": ["node"]
},
"include": ["vite.config.ts", "tailwind.config.ts"]
"include": ["vite.config.ts", "tailwind.config.ts", "vite-plugin-metrics.ts"]
}
+42
View File
@@ -0,0 +1,42 @@
import type { Plugin } from 'vite';
/**
* Vite plugin that exposes a /metrics endpoint during development.
*
* The endpoint returns Prometheus‑text metrics collected in
* src/lib/telemetry.ts.
*/
export function metricsPlugin(): Plugin {
let telemetryModule: typeof import('./src/lib/telemetry') | null = null;
return {
name: 'zeavis-metrics',
configureServer(server) {
server.middlewares.use(async (req, res, next) => {
// Only handle GET /metrics
if (req.method !== 'GET' || !req.url?.startsWith('/metrics')) {
return next();
}
// Lazy‑load the telemetry module (ensures the app is bootstrapped first)
if (!telemetryModule) {
try {
telemetryModule = await server.ssrLoadModule('./src/lib/telemetry.ts') as typeof import('./src/lib/telemetry');
} catch {
// If the module isn't ready yet, return an empty body
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain; charset=utf-8');
res.end('# telemetry module not yet loaded\n');
return;
}
}
const body = telemetryModule.collectMetrics();
res.statusCode = 200;
res.setHeader('Content-Type', 'text/plain; charset=utf-8');
res.end(body);
});
},
};
}
+2 -1
View File
@@ -2,13 +2,14 @@ import react from '@vitejs/plugin-react';
import tsconfigPaths from 'vite-tsconfig-paths';
import path from 'node:path';
import { defineConfig, loadEnv } from 'vite';
import { metricsPlugin } from './vite-plugin-metrics';
export default defineConfig(({ mode }) => {
const env = loadEnv(mode, process.cwd(), '');
const apiProxyTarget = env.VITE_API_PROXY_TARGET || 'http://localhost:3000';
return {
plugins: [react(), tsconfigPaths()],
plugins: [react(), tsconfigPaths(), metricsPlugin()],
server: {
proxy: {
'/api': apiProxyTarget,
+234
View File
File diff suppressed because one or more lines are too long
+28
View File
@@ -0,0 +1,28 @@
# =============================================================================
# ZeaVis Edu — Telemetry Stack Integration (LOCAL DEV ONLY)
#
# This override connects the Telemetry submodule docker-compose to the same
# Docker network as ZeaVis Edu services for LOCAL development on a single host.
#
# In PRODUCTION, the app and telemetry run on separate VPS instances
# connected via Tailscale. See telemetry/prometheus/targets/zeavis-edu.json
# for the Tailscale IP configuration.
#
# Usage (local dev only):
# # Ensure app-shared-net exists first:
# docker network create app-shared-net
#
# # Start telemetry with app network access:
# make telemetry-up-local
# =============================================================================
networks:
app-shared-net:
external: true
name: app-shared-net
services:
prometheus:
networks:
- default
- app-shared-net
+27
View File
@@ -2,6 +2,9 @@ networks:
app-shared-net:
external: true
name: app-shared-net
telemetry-net:
external: true
name: telemetry-net
services:
web:
@@ -10,6 +13,7 @@ services:
restart: always
networks:
- app-shared-net
- telemetry-net
env_file:
- .env
labels:
@@ -26,6 +30,7 @@ services:
restart: always
networks:
- app-shared-net
- telemetry-net
env_file:
- .env
environment:
@@ -33,6 +38,8 @@ services:
API_PORT: "3000"
WEB_APP_URL: https://zeavisedu.asepharyana.my.id
ML_SERVICE_URL: ${ML_SERVICE_URL:-http://zeavis-ml:8000}
ports:
- "3000:3000"
labels:
traefik.enable: "true"
traefik.http.routers.zeavis-api.rule: Host(`api-zeavisedu.asepharyana.my.id`)
@@ -41,17 +48,37 @@ services:
traefik.http.routers.zeavis-api.tls.certresolver: cloudflare
traefik.http.services.zeavis-api.loadbalancer.server.port: "3000"
# Node Exporter — expose system metrics (CPU, RAM, disk) for Prometheus scraping
node_exporter:
image: prom/node-exporter:v1.8.2
container_name: zeavis-node-exporter
restart: unless-stopped
ports:
- "9100:9100"
command:
- "--path.rootfs=/host"
- "--web.listen-address=:9100"
volumes:
- /:/host:ro,rslave
pid: host
networks:
- app-shared-net
- telemetry-net
ml:
image: ghcr.io/${GITHUB_REPOSITORY:-mytheclipse/zeavis-edu}/ml:main
container_name: zeavis-ml
restart: always
networks:
- app-shared-net
- telemetry-net
env_file:
- .env
environment:
MODEL_PATH: /app/model/model.onnx
MODEL_INPUT_SIZE: "224"
ports:
- "8000:8000"
labels:
traefik.enable: "true"
traefik.http.routers.zeavis-ml.rule: Host(`ml-zeavisedu.asepharyana.my.id`)
+140
View File
@@ -0,0 +1,140 @@
# Infra — ZeaVis Edu Multi-VPS Deployment
## Arsitektur
```
┌─────────────────────────────────────────────┐ ┌──────────────────────────────────────────────┐
│ App VPS (imrnes) │ │ Telemetry VPS (orange) │
│ 100.108.1.124 │ │ 100.96.248.86 │
│ Arch Linux │ │ Ubuntu │
│ │ │ │
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ ┌──────────┐ ┌──────────────┐ │
│ │ Web │ │ API │ │ ML │ │ │ │Prometheus│ │Metric │ │
│ │:80 │ │:3000 │ │:8000 │ │ │ │:9090 │ │Ingester │ │
│ │/metrics │ │/metrics │ │/metrics │ │ │ │ │ │:9091 │ │
│ └──────────┘ └──────────┘ └──────────┘ │ │ └────┬─────┘ └──────┬───────┘ │
│ ┌──────────────────────────────────────┐ │ │ │ │ │
│ │ Node Exporter │ │ │ │ remote_write │ │
│ │ :9100 │ │ │ ▼ ▼ │
│ └──────────────────────────────────────┘ │ │ ┌──────────────────────────────────────┐ │
│ │ │ │ Vector │ │
│ ┌──────────────┐ │ │ │ :9001 │ │
│ │ Traefik │ │ │ └────────────────┬─────────────────────┘ │
│ │ (Coolify) │ │ │ │ │
│ └──────────────┘ │ │ ▼ │
│ │ │ ┌──────────────────────────────────────┐ │
│ ZeaVis Edu Apps via │ │ │ ClickHouse │ │
│ zeavisedu.asepharyana.my.id │ │ │ :8123 │ │
│ │ │ └───────────────┬──────────────────────┘ │
│ │ │ │ │
│ │ │ ▼ │
│ ==== Tailscale (WireGuard) ==== │ │ ┌──────────────────────────────────────┐ │
│ │ │ │ Query Proxy │ │
│ │ │ │ :9092 │ │
│ │ │ └───────────────┬──────────────────────┘ │
│ │ │ │ │
│ │ │ ▼ │
│ │ │ ┌──────────────────────────────────────┐ │
│ │ │ │ Telemetry UI (nginx) │ │
│ │ │ │ :8181 │ │
│ │ │ └──────────────────────────────────────┘ │
│ │ │ │
│ │ │ Coolify + Traefik handles: │
│ │ │ telemetry.zeavisedu.asepharyana.my.id │
└─────────────────────────────────────────────┘ └──────────────────────────────────────────────┘
```
## Prerequisites
### GitHub Secrets (untuk CI/CD)
**App VPS deploy (`.github/workflows/deploy.yml`):**
| Secret | Value |
|--------|-------|
| `VPS_HOST` | `100.108.1.124` (imrnes) |
| `VPS_USER` | `mytheclipse` |
| `VPS_SSH_KEY` | Private SSH key for imrnes |
| `VPS_PORT` | `22` |
| `DATABASE_URL` | PostgreSQL connection string |
| `SESSION_SECRET` | Random session secret |
**Telemetry VPS deploy (`.github/workflows/telemetry-ci-cd.yml`):**
| Secret | Value |
|--------|-------|
| `TELEMETRY_VPS_HOST` | `100.96.248.86` (orange) |
| `TELEMETRY_VPS_USER` | SSH username for orange |
| `TELEMETRY_VPS_SSH_KEY` | Private SSH key for orange |
| `TELEMETRY_VPS_PORT` | `22` |
| `GHCR_PAT` | GitHub PAT with `write:packages` + `read:packages` |
### VPS Setup
#### 1. App VPS (imrnes — 100.108.1.124)
```bash
# Create Docker network
docker network create app-shared-net
docker network create telemetry-net
# ZeaVis Edu apps deploy automatically via GitHub Actions
```
#### 2. Telemetry VPS (orange — 100.96.248.86)
Deploy via GitHub Actions workflow `.github/workflows/telemetry-ci-cd.yml`.
Atau manual:
```bash
ssh mytheclipse@100.96.248.86
mkdir -p /opt/telemetry
# ... sync files from telemetry/ directory ...
cd /opt/telemetry
docker compose up -d
bash clickhouse/init.sh
```
## Port yang dibuka
### App VPS (imrnes)
| Port | Service | Akses |
|------|---------|-------|
| 80/443 | Web (via Traefik/Coolify) | Public |
| 3000 | API metrics | Tailscale-only |
| 8000 | ML service metrics | Tailscale-only |
| 9100 | Node Exporter | Tailscale-only |
### Telemetry VPS (orange)
| Port | Service | Akses |
|------|---------|-------|
| 80/443 | Telemetry UI (via Coolify Traefik) | Public |
| 8181 | Telemetry UI (direct) | Tailscale-only |
| 9090 | Prometheus | Tailscale-only |
| 9091 | Metric Ingester | Tailscale-only |
| 9001 | Vector HTTP source | Tailscale-only |
| 8123 | ClickHouse HTTP | Tailscale-only |
| 9000 | ClickHouse Native | Tailscale-only |
## Metrics Flow
1. **App services** expose `/metrics` pada port masing-masing
2. **Prometheus** di orange VPS scrape via Tailscale IP (`100.108.1.124:PORT`)
3. **Prometheus** forward ke **Metric Ingester** via `remote_write`
4. **Metric Ingester** enrich → filter → forward ke **Vector**
5. **Vector** buffer → write ke **ClickHouse**
6. **Telemetry UI** query via **Query Proxy** → **ClickHouse**
## Useful Commands
```bash
# Telemetry stack status
make telemetry-status
# View telemetry logs
make telemetry-logs s=prometheus
# Send test metric
make telemetry-test-metric
# Restart a service
make telemetry-restart s=vector
```
Submodule
+1
Submodule telemetry added at 2582a53592