Add MiniCPM-V 4.6 Benchmark Notebook for model evaluation and performance metrics
This commit is contained in:
asepharyana committed
commit
c58c4bca03
3 files changed
+1067
No files matched your search
@@ -0,0 +1,28 @@
|
|||||||
|
---
|
||||||
|
mode: primary
|
||||||
|
description: Run notebook-first data analysis by appending and executing cells
|
||||||
|
for each request.
|
||||||
|
options:
|
||||||
|
displayName: Data
|
||||||
|
id: data
|
||||||
|
requirements:
|
||||||
|
skills:
|
||||||
|
- data-investigation
|
||||||
|
vscode_extensions:
|
||||||
|
- name: Jupyter
|
||||||
|
id: ms-toolsai.jupyter
|
||||||
|
color: "#2563EB"
|
||||||
|
---
|
||||||
|
|
||||||
|
You are Kilo, a notebook-first data analysis agent. Use an active Jupyter notebook as the working surface.
|
||||||
|
|
||||||
|
Guidelines:
|
||||||
|
- If no notebook is active, create a uniquely named, descriptive `<topic>.ipynb` in the current workspace folder
|
||||||
|
- Use the dedicated notebook tools to create, read, edit, and execute; prefer these tools over other methods like MCP tools and manual raw JSON editing
|
||||||
|
- Confirm Jupyter and kernel readiness through the first requested notebook execution; only notify the user if they need to select or configure a kernel before work can continue
|
||||||
|
- For every user request, append at least one focused code cell and execute it
|
||||||
|
- Preserve notebook history: do not modify or delete existing cells unless explicitly asked; after failures, append diagnostic or corrected cells
|
||||||
|
- Keep substantive data work and supporting evidence in the notebook
|
||||||
|
- Avoid changing non-notebook files unless explicitly requested or necessary to complete the task
|
||||||
|
- Inspect cell output before answering, and keep notebook outputs and final summaries concise
|
||||||
|
- Never claim execution when a notebook cell did not run
|
||||||
@@ -0,0 +1,616 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Gemma 4 E4B — Clone & Benchmark Notebook\n",
|
||||||
|
"\n",
|
||||||
|
"Notebook untuk meng-clone model [google/gemma-4-E4B](https://huggingface.co/google/gemma-4-E4B) dari Hugging Face dan melakukan benchmark pada berbagai metrik:\n",
|
||||||
|
"- Kecepatan loading & memory usage\n",
|
||||||
|
"- Text generation throughput (tokens/sec)\n",
|
||||||
|
"- Reasoning & knowledge QA\n",
|
||||||
|
"- Coding capability\n",
|
||||||
|
"- Multimodal understanding (image)\n",
|
||||||
|
"- Long context retrieval\n",
|
||||||
|
"\n",
|
||||||
|
"**Model**: `google/gemma-4-E4B-it` (instruction-tuned, 4.5B effective params, 8B total, 128K context)\n",
|
||||||
|
"\n",
|
||||||
|
"**Cara pakai**: Runtime > Factory reset runtime, lalu Runtime > Run all"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 1. Environment Setup"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 7,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 60.7/60.7 MB 13.8 MB/s eta 0:00:00\n",
|
||||||
|
"Install selesai\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# Install dependencies\n",
|
||||||
|
"!pip install -qU \\\n",
|
||||||
|
" 'transformers>=4.50.0' \\\n",
|
||||||
|
" accelerate \\\n",
|
||||||
|
" sentencepiece \\\n",
|
||||||
|
" protobuf \\\n",
|
||||||
|
" psutil \\\n",
|
||||||
|
" 'pillow<11' \\\n",
|
||||||
|
" requests \\\n",
|
||||||
|
" matplotlib \\\n",
|
||||||
|
" tabulate \\\n",
|
||||||
|
" librosa \\\n",
|
||||||
|
" soundfile \\\n",
|
||||||
|
" einops \\\n",
|
||||||
|
" bitsandbytes \\\n",
|
||||||
|
" 2>&1 | tail -3\n",
|
||||||
|
"print(\"Install selesai\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Python : 3.12.13 (main, Mar 4 2026, 09:23:07) [GCC 11.4.0]\n",
|
||||||
|
"PyTorch : 2.11.0+cu128\n",
|
||||||
|
"CUDA avail : True\n",
|
||||||
|
"CUDA device : Tesla T4\n",
|
||||||
|
"CUDA VRAM : 15.6 GB\n",
|
||||||
|
"CUDA cap : (7, 5)\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"import os, sys, json, time, gc, warnings\n",
|
||||||
|
"from pathlib import Path\n",
|
||||||
|
"from datetime import datetime\n",
|
||||||
|
"from IPython.display import display\n",
|
||||||
|
"\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import psutil\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"from tabulate import tabulate\n",
|
||||||
|
"from PIL import Image\n",
|
||||||
|
"import requests\n",
|
||||||
|
"from io import BytesIO\n",
|
||||||
|
"\n",
|
||||||
|
"warnings.filterwarnings(\"ignore\")\n",
|
||||||
|
"\n",
|
||||||
|
"print(f\"Python : {sys.version}\")\n",
|
||||||
|
"print(f\"PyTorch : {torch.__version__}\")\n",
|
||||||
|
"print(f\"CUDA avail : {torch.cuda.is_available()}\")\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" print(f\"CUDA device : {torch.cuda.get_device_name(0)}\")\n",
|
||||||
|
" print(f\"CUDA VRAM : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n",
|
||||||
|
" print(f\"CUDA cap : {torch.cuda.get_device_capability()}\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 2. Clone Model from Hugging Face\n",
|
||||||
|
"\n",
|
||||||
|
"Model size ~16 GB dalam BF16. Karena T4 hanya 15.6GB VRAM, kita perlu 4-bit quantization."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Model ID: google/gemma-4-E4B-it\n",
|
||||||
|
"Loading (4-bit quantized)...\n",
|
||||||
|
"Processor loaded in 6.2s\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"application/vnd.jupyter.widget-view+json": {
|
||||||
|
"model_id": "169a4541ec42498083b317e11576b863",
|
||||||
|
"version_major": 2,
|
||||||
|
"version_minor": 0
|
||||||
|
},
|
||||||
|
"text/plain": [
|
||||||
|
"Loading weights: 0%| | 0/2076 [00:00<?, ?it/s]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"\n",
|
||||||
|
"Model loaded in 63.4s\n",
|
||||||
|
"Parameters: 5.72B\n",
|
||||||
|
"Device: cuda:0, Dtype: torch.bfloat16\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"MODEL_ID = \"google/gemma-4-E4B-it\"\n",
|
||||||
|
"CACHE_DIR = None\n",
|
||||||
|
"\n",
|
||||||
|
"print(f\"Model ID: {MODEL_ID}\")\n",
|
||||||
|
"print(\"Loading (4-bit quantized)...\")\n",
|
||||||
|
"t0 = time.perf_counter()\n",
|
||||||
|
"\n",
|
||||||
|
"from transformers import AutoProcessor, AutoModelForMultimodalLM, BitsAndBytesConfig\n",
|
||||||
|
"import accelerate\n",
|
||||||
|
"\n",
|
||||||
|
"bnb_config = BitsAndBytesConfig(\n",
|
||||||
|
" load_in_4bit=True,\n",
|
||||||
|
" bnb_4bit_compute_dtype=torch.bfloat16,\n",
|
||||||
|
" bnb_4bit_use_double_quant=True,\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"processor = AutoProcessor.from_pretrained(MODEL_ID, cache_dir=CACHE_DIR)\n",
|
||||||
|
"print(f\"Processor loaded in {time.perf_counter()-t0:.1f}s\")\n",
|
||||||
|
"\n",
|
||||||
|
"load_start = time.perf_counter()\n",
|
||||||
|
"model = AutoModelForMultimodalLM.from_pretrained(\n",
|
||||||
|
" MODEL_ID,\n",
|
||||||
|
" torch_dtype=torch.bfloat16,\n",
|
||||||
|
" device_map={\"\": \"cuda:0\"},\n",
|
||||||
|
" max_memory={0: \"14GiB\", \"cpu\": \"48GiB\"},\n",
|
||||||
|
" cache_dir=CACHE_DIR,\n",
|
||||||
|
" quantization_config=bnb_config,\n",
|
||||||
|
")\n",
|
||||||
|
"load_time = time.perf_counter() - load_start\n",
|
||||||
|
"print(f\"\\nModel loaded in {load_time:.1f}s\")\n",
|
||||||
|
"\n",
|
||||||
|
"total_params = sum(p.numel() for p in model.parameters())\n",
|
||||||
|
"print(f\"Parameters: {total_params/1e9:.2f}B\")\n",
|
||||||
|
"print(f\"Device: {model.device}, Dtype: {model.dtype}\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 11,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"VRAM allocated: 9.32 GB\n",
|
||||||
|
"RAM used: 2.67 GB\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" vram_used = torch.cuda.memory_allocated() / 1e9\n",
|
||||||
|
" print(f\"VRAM allocated: {vram_used:.2f} GB\")\n",
|
||||||
|
"ram_used = psutil.Process(os.getpid()).memory_info().rss / 1e9\n",
|
||||||
|
"print(f\"RAM used: {ram_used:.2f} GB\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 3. Text Generation Throughput"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 12,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Simple QA: 42 tok, 7.5s, 5.2 tok/s\n",
|
||||||
|
"Math: 217 tok, 37.3s, 5.8 tok/s\n",
|
||||||
|
"Code: 256 tok, 43.7s, 5.9 tok/s\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"def throughput(prompt, max_tokens=256, runs=3):\n",
|
||||||
|
" msgs = [{\"role\": \"user\", \"content\": prompt}]\n",
|
||||||
|
" inputs = processor.apply_chat_template(msgs, tokenize=True, return_dict=True,\n",
|
||||||
|
" return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
|
||||||
|
" inp_len = inputs[\"input_ids\"].shape[-1]\n",
|
||||||
|
" lats, toks = [], []\n",
|
||||||
|
" for _ in range(runs):\n",
|
||||||
|
" start = time.perf_counter()\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out = model.generate(**inputs, max_new_tokens=max_tokens, do_sample=True, temperature=0.7)\n",
|
||||||
|
" elapsed = time.perf_counter() - start\n",
|
||||||
|
" gen = out[0][inp_len:]\n",
|
||||||
|
" lats.append(elapsed)\n",
|
||||||
|
" toks.append(len(gen))\n",
|
||||||
|
" tps = [t/l for t,l in zip(toks, lats)]\n",
|
||||||
|
" return {\"prompt\": prompt[:60]+\"...\", \"tokens\": int(np.mean(toks)),\n",
|
||||||
|
" \"latency\": float(np.mean(lats)), \"tps\": float(np.mean(tps))}\n",
|
||||||
|
"\n",
|
||||||
|
"prompts = {\n",
|
||||||
|
" \"Simple QA\": \"What is the capital of Indonesia?\",\n",
|
||||||
|
" \"Math\": \"If a train travels at 120 km/h and another at 80 km/h toward each other from 500 km apart, how long until they meet?\",\n",
|
||||||
|
" \"Code\": \"Write a Python function to find the longest palindromic substring.\",\n",
|
||||||
|
"}\n",
|
||||||
|
"\n",
|
||||||
|
"results = []\n",
|
||||||
|
"for name, p in prompts.items():\n",
|
||||||
|
" r = throughput(p, runs=2)\n",
|
||||||
|
" results.append(r)\n",
|
||||||
|
" print(f\"{name}: {r['tokens']} tok, {r['latency']:.1f}s, {r['tps']:.1f} tok/s\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 4. Reasoning (MMLU-style)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 13,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
" OK Expected=B Got=B | What is the time complexity of binary search?\n",
|
||||||
|
" OK Expected=C Got=C | Which planet has the strongest surface gravity?\n",
|
||||||
|
" OK Expected=C Got=C | In C++, which keyword prevents overriding?\n",
|
||||||
|
" NO Expected=B Got=A | Probability of drawing a red ball from 3 red + 5 b\n",
|
||||||
|
" OK Expected=B Got=B | What does mitochondria do?\n",
|
||||||
|
"\n",
|
||||||
|
"Accuracy: 4/5 = 80%\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"mmlu = [\n",
|
||||||
|
" {\"q\": \"What is the time complexity of binary search?\", \"o\": [\"A. O(n)\", \"B. O(log n)\", \"C. O(n log n)\", \"D. O(1)\"], \"a\": \"B\"},\n",
|
||||||
|
" {\"q\": \"Which planet has the strongest surface gravity?\", \"o\": [\"A. Earth\", \"B. Mars\", \"C. Jupiter\", \"D. Saturn\"], \"a\": \"C\"},\n",
|
||||||
|
" {\"q\": \"In C++, which keyword prevents overriding?\", \"o\": [\"A. static\", \"B. const\", \"C. final\", \"D. override\"], \"a\": \"C\"},\n",
|
||||||
|
" {\"q\": \"Probability of drawing a red ball from 3 red + 5 blue?\", \"o\": [\"A. 3/5\", \"B. 3/8\", \"C. 5/8\", \"D. 1/2\"], \"a\": \"B\"},\n",
|
||||||
|
" {\"q\": \"What does mitochondria do?\", \"o\": [\"A. Protein\", \"B. Energy (ATP)\", \"C. Lipid\", \"D. DNA\"], \"a\": \"B\"},\n",
|
||||||
|
"]\n",
|
||||||
|
"\n",
|
||||||
|
"ok = 0\n",
|
||||||
|
"for q in mmlu:\n",
|
||||||
|
" prompt = f\"{q['q']}\\n\\n\" + \"\\n\".join(q[\"o\"]) + \"\\n\\nAnswer with a single letter:\"\n",
|
||||||
|
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
|
||||||
|
" tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out = model.generate(**inputs, max_new_tokens=8, do_sample=False)\n",
|
||||||
|
" ans = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n",
|
||||||
|
" cor = q[\"a\"] in ans.upper()[:1]\n",
|
||||||
|
" if cor: ok += 1\n",
|
||||||
|
" print(f\" {'OK' if cor else 'NO'} Expected={q['a']} Got={ans[:20]} | {q['q'][:50]}\")\n",
|
||||||
|
"\n",
|
||||||
|
"print(f\"\\nAccuracy: {ok}/{len(mmlu)} = {ok/len(mmlu)*100:.0f}%\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 5. Coding"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 14,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"[transformers] The following generation flags are not valid and may be ignored: ['temperature']. Set `TRANSFORMERS_VERBOSITY=info` for more details.\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"\n",
|
||||||
|
"========================================\n",
|
||||||
|
"Binary Search\n",
|
||||||
|
"========================================\n",
|
||||||
|
"```python\n",
|
||||||
|
"def binary_search(arr, target):\n",
|
||||||
|
" \"\"\"\n",
|
||||||
|
" Performs a binary search on a sorted array to find the target element.\n",
|
||||||
|
"\n",
|
||||||
|
" Args:\n",
|
||||||
|
" arr: A sorted list of elements (the array to search).\n",
|
||||||
|
" target: The element whose index is to be found.\n",
|
||||||
|
"\n",
|
||||||
|
" Returns:\n",
|
||||||
|
" The index of the target if found, otherwise -1.\n",
|
||||||
|
" \"\"\"\n",
|
||||||
|
" left = 0\n",
|
||||||
|
" right = len(arr) - 1\n",
|
||||||
|
"\n",
|
||||||
|
" while left <= right:\n",
|
||||||
|
" # \n",
|
||||||
|
"\n",
|
||||||
|
"========================================\n",
|
||||||
|
"Fibonacci\n",
|
||||||
|
"========================================\n",
|
||||||
|
"Here are several ways to implement the Fibonacci sequence function `fib(n)` using Dynamic Programming (DP) in Python, depending on whether you want to optimize for space or time complexity.\n",
|
||||||
|
"\n",
|
||||||
|
"The standard Fibonacci sequence starts with $F_0 = 0$ and $F_1 = 1$.\n",
|
||||||
|
"\n",
|
||||||
|
"---\n",
|
||||||
|
"\n",
|
||||||
|
"## 1. Top-Down DP with Memoization (Recursive with Caching)\n",
|
||||||
|
"\n",
|
||||||
|
"This is the most direct translation of applying DP to the recursive defin\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"for name, prompt in [\n",
|
||||||
|
" (\"Binary Search\", \"Write Python `binary_search(arr, target)` returning index or -1.\"),\n",
|
||||||
|
" (\"Fibonacci\", \"Write Python `fib(n)` for nth Fibonacci using DP.\"),\n",
|
||||||
|
"]:\n",
|
||||||
|
" print(f\"\\n{'='*40}\\n{name}\\n{'='*40}\")\n",
|
||||||
|
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
|
||||||
|
" tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out = model.generate(**inputs, max_new_tokens=512, temperature=0.2, do_sample=False)\n",
|
||||||
|
" print(processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)[:400])"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 6. Image Understanding"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 15,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAC0CAYAAAC9vjEhAAEAAElEQVR4AXT9V5Bu23bfh83OvXd375zPPjnegHMRLoCLi0AQJAFSNASCIinJskRZKrv8IMnlsvXgB7v8YJdt2VV+U8mUZEklybRFCyQYRBskCBAZuPncfHLaOe/dvXfn9u/3H3N+X59LanV/a80w0hxzzDzXXDP//n/4Tw8OZlo7aHXhxD3DHyFEzMxUiDDtYJbw/cQKJY6QXuJMrsPOIIIzI/QsPymEdDBmi0hoSTFsuJcE8uqXsoiNPAclWvziSEJP4vHMwGdfWgQkjmglFy9y8JTTfmQiTiDcI42SO8AvRyUOkdBXcrkY1CnDZB/ESlmDZulnhrCCFg7uoiGAPOP0TlRSVQElr3KEtPIXTEki7EHbI8jnbNeH6SyKRgh/0OYMwbmnoPLpECP3DoCZlU+PUVb+40t6g1PpUJYDiMlTEH+Line truncated
|
||||||
|
"text/plain": [
|
||||||
|
"<PIL.Image.Image image mode=RGBA size=250x180>"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Time: 23.8s\n",
|
||||||
|
"This image displays a photograph featuring the **Golden Gate Bridge** spanning across a body of water toward a landmass in the distance.\n",
|
||||||
|
"\n",
|
||||||
|
"Here is a brief description:\n",
|
||||||
|
"\n",
|
||||||
|
"The dominant feature is the massive **red bridge structure** (clearly identifiable\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"try:\n",
|
||||||
|
" url = \"https://raw.githubusercontent.com/google-gemma/cookbook/main/apps/sample-data/GoldenGate.png\"\n",
|
||||||
|
" img = Image.open(BytesIO(requests.get(url, timeout=30).content))\n",
|
||||||
|
" display(img.resize((250, 180)))\n",
|
||||||
|
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":[\n",
|
||||||
|
" {\"type\":\"image\",\"image\":img},\n",
|
||||||
|
" {\"type\":\"text\",\"text\":\"What is shown? Describe briefly.\"}\n",
|
||||||
|
" ]}], tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
|
||||||
|
" t0 = time.perf_counter()\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out = model.generate(**inputs, max_new_tokens=128, temperature=0.7, do_sample=True)\n",
|
||||||
|
" print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n",
|
||||||
|
" print(processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)[:250])\n",
|
||||||
|
"except Exception as e:\n",
|
||||||
|
" print(f\"ERROR: {e}\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 7. Long Context (Needle-in-Haystack)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 17,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Needle-in-Haystack:\n",
|
||||||
|
"EARLY: 793 tok, correct=OK, 4.9s\n",
|
||||||
|
"MIDDLE: 793 tok, correct=OK, 5.0s\n",
|
||||||
|
"LATE: 793 tok, correct=OK, 5.0s\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"torch.cuda.empty_cache()\n",
|
||||||
|
"gc.collect()\n",
|
||||||
|
"\n",
|
||||||
|
"def needle(pos):\n",
|
||||||
|
" needle_str = \"The secret code is BLUE-42-GREEN.\"\n",
|
||||||
|
" filler = \"The quick brown fox jumps over the lazy dog. Python is versatile. \"\n",
|
||||||
|
" sents = [filler] * 50 # shorter context for T4\n",
|
||||||
|
" if pos == \"early\": sents.insert(0, needle_str)\n",
|
||||||
|
" elif pos == \"middle\": sents.insert(len(sents)//2, needle_str)\n",
|
||||||
|
" else: sents.append(needle_str)\n",
|
||||||
|
" prompt = f\"Read the text and answer.\\n\\nText: {' '.join(sents)}\\n\\nQ: What is the secret code? Answer with code only.\"\n",
|
||||||
|
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
|
||||||
|
" tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
|
||||||
|
" t0 = time.perf_counter()\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out = model.generate(**inputs, max_new_tokens=16, do_sample=False)\n",
|
||||||
|
" resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n",
|
||||||
|
" return {\"pos\": pos, \"tokens\": inputs[\"input_ids\"].shape[-1], \"resp\": resp[:80],\n",
|
||||||
|
" \"correct\": \"BLUE-42-GREEN\" in resp, \"time\": time.perf_counter()-t0}\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"Needle-in-Haystack:\")\n",
|
||||||
|
"for p in [\"early\", \"middle\", \"late\"]:\n",
|
||||||
|
" r = needle(p)\n",
|
||||||
|
" print(f\"{p.upper()}: {r['tokens']} tok, correct={'OK' if r['correct'] else 'NO'}, {r['time']:.1f}s\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 8. Summary"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 18,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"============================================================\n",
|
||||||
|
" GEMMA 4 E4B — SUMMARY\n",
|
||||||
|
"============================================================\n",
|
||||||
|
"+------------+-----------------------+\n",
|
||||||
|
"| Metric | Value |\n",
|
||||||
|
"+============+=======================+\n",
|
||||||
|
"| Model | google/gemma-4-E4B-it |\n",
|
||||||
|
"+------------+-----------------------+\n",
|
||||||
|
"| Parameters | 5.72B |\n",
|
||||||
|
"+------------+-----------------------+\n",
|
||||||
|
"| Device | cuda:0 |\n",
|
||||||
|
"+------------+-----------------------+\n",
|
||||||
|
"| VRAM | 10.26 GB |\n",
|
||||||
|
"+------------+-----------------------+\n",
|
||||||
|
"| RAM | 2.67 GB |\n",
|
||||||
|
"+------------+-----------------------+\n",
|
||||||
|
"| Load Time | 63.4s |\n",
|
||||||
|
"+------------+-----------------------+\n",
|
||||||
|
"| Throughput | 5.6 tok/s |\n",
|
||||||
|
"+------------+-----------------------+\n",
|
||||||
|
"| MMLU | 4/5 (80%) |\n",
|
||||||
|
"+------------+-----------------------+\n",
|
||||||
|
"============================================================\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"rows = [\n",
|
||||||
|
" [\"Model\", MODEL_ID],\n",
|
||||||
|
" [\"Parameters\", f\"{total_params/1e9:.2f}B\"],\n",
|
||||||
|
" [\"Device\", str(model.device)],\n",
|
||||||
|
"]\n",
|
||||||
|
"if torch.cuda.is_available(): rows.append([\"VRAM\", f\"{torch.cuda.memory_allocated()/1e9:.2f} GB\"])\n",
|
||||||
|
"rows.append([\"RAM\", f\"{ram_used:.2f} GB\"])\n",
|
||||||
|
"rows.append([\"Load Time\", f\"{load_time:.1f}s\"])\n",
|
||||||
|
"if results: rows.append([\"Throughput\", f\"{np.mean([r['tps'] for r in results]):.1f} tok/s\"])\n",
|
||||||
|
"rows.append([\"MMLU\", f\"{ok}/{len(mmlu)} ({ok/len(mmlu)*100:.0f}%)\"])\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"=\"*60)\n",
|
||||||
|
"print(\" GEMMA 4 E4B — SUMMARY\")\n",
|
||||||
|
"print(\"=\"*60)\n",
|
||||||
|
"print(tabulate(rows, headers=[\"Metric\",\"Value\"], tablefmt=\"grid\"))\n",
|
||||||
|
"print(\"=\"*60)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 9. Cleanup"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 19,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"VRAM: 10.26 GB\n",
|
||||||
|
"Done.\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"del model, processor\n",
|
||||||
|
"gc.collect()\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.cuda.empty_cache()\n",
|
||||||
|
" print(f\"VRAM: {torch.cuda.memory_allocated()/1e9:.2f} GB\")\n",
|
||||||
|
"print(\"Done.\")"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.12.13"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
@@ -0,0 +1,423 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# MiniCPM-V 4.6 — Clone & Benchmark Notebook\n",
|
||||||
|
"\n",
|
||||||
|
"Notebook untuk meng-clone model [openbmb/MiniCPM-V-4.6](https://huggingface.co/openbmb/MiniCPM-V-4.6) dari Hugging Face dan melakukan benchmark:\n",
|
||||||
|
"- Loading speed & memory usage\n",
|
||||||
|
"- Text generation throughput (tokens/sec)\n",
|
||||||
|
"- Reasoning & knowledge QA\n",
|
||||||
|
"- Coding capability\n",
|
||||||
|
"- Multimodal image understanding\n",
|
||||||
|
"- Long context (needle-in-haystack)\n",
|
||||||
|
"\n",
|
||||||
|
"**Model**: `openbmb/MiniCPM-V-4.6` (SigLIP2-400M + Qwen3.5-0.8B, 1B total params)\n",
|
||||||
|
"\n",
|
||||||
|
"**Cara pakai**: Runtime > Factory reset runtime, lalu Runtime > Run all"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 1. Environment Setup"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Install dependencies — skip torch, Colab sudah punya\n",
|
||||||
|
"!pip install -qU \\\n",
|
||||||
|
" 'transformers>=5.7.0' \\\n",
|
||||||
|
" accelerate \\\n",
|
||||||
|
" sentencepiece \\\n",
|
||||||
|
" psutil \\\n",
|
||||||
|
" 'pillow<11' \\\n",
|
||||||
|
" requests \\\n",
|
||||||
|
" matplotlib \\\n",
|
||||||
|
" tabulate \\\n",
|
||||||
|
" av \\\n",
|
||||||
|
" einops \\\n",
|
||||||
|
" 2>&1 | tail -3\n",
|
||||||
|
"print(\"Install selesai\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os, sys, json, time, gc, warnings\n",
|
||||||
|
"from pathlib import Path\n",
|
||||||
|
"from datetime import datetime\n",
|
||||||
|
"from IPython.display import display, Video\n",
|
||||||
|
"\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import psutil\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"from tabulate import tabulate\n",
|
||||||
|
"from PIL import Image\n",
|
||||||
|
"import requests\n",
|
||||||
|
"from io import BytesIO\n",
|
||||||
|
"\n",
|
||||||
|
"warnings.filterwarnings(\"ignore\")\n",
|
||||||
|
"\n",
|
||||||
|
"print(f\"Python : {sys.version}\")\n",
|
||||||
|
"print(f\"PyTorch : {torch.__version__}\")\n",
|
||||||
|
"print(f\"CUDA avail : {torch.cuda.is_available()}\")\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" print(f\"CUDA device : {torch.cuda.get_device_name(0)}\")\n",
|
||||||
|
" print(f\"CUDA VRAM : {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\n",
|
||||||
|
" print(f\"CUDA cap : {torch.cuda.get_device_capability()}\")\n",
|
||||||
|
"\n",
|
||||||
|
"try:\n",
|
||||||
|
" import transformers\n",
|
||||||
|
" print(f\"transformers : {transformers.__version__}\")\n",
|
||||||
|
"except:\n",
|
||||||
|
" print(\"transformers : NOT INSTALLED\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 2. Clone Model from Hugging Face\n",
|
||||||
|
"\n",
|
||||||
|
"MiniCPM-V 4.6 = 1B params. Muat di T4 (15.6GB) tanpa masalah."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"MODEL_ID = \"openbmb/MiniCPM-V-4.6\"\n",
|
||||||
|
"CACHE_DIR = None # Set ke \"drive/MyDrive/hf_cache\" untuk persistent\n",
|
||||||
|
"DOWNSAMPLE_MODE = \"16x\" # \"16x\" = efisien, \"4x\" = detail lebih tinggi\n",
|
||||||
|
"\n",
|
||||||
|
"print(f\"Model ID: {MODEL_ID}\")\n",
|
||||||
|
"print(\"Loading...\")\n",
|
||||||
|
"t0 = time.perf_counter()\n",
|
||||||
|
"\n",
|
||||||
|
"from transformers import AutoProcessor, AutoModelForImageTextToText\n",
|
||||||
|
"\n",
|
||||||
|
"processor = AutoProcessor.from_pretrained(MODEL_ID, cache_dir=CACHE_DIR)\n",
|
||||||
|
"print(f\"Processor loaded in {time.perf_counter()-t0:.1f}s\")\n",
|
||||||
|
"\n",
|
||||||
|
"load_start = time.perf_counter()\n",
|
||||||
|
"model = AutoModelForImageTextToText.from_pretrained(\n",
|
||||||
|
" MODEL_ID,\n",
|
||||||
|
" torch_dtype=torch.bfloat16,\n",
|
||||||
|
" device_map=\"auto\",\n",
|
||||||
|
" cache_dir=CACHE_DIR,\n",
|
||||||
|
")\n",
|
||||||
|
"load_time = time.perf_counter() - load_start\n",
|
||||||
|
"print(f\"\\nModel loaded in {load_time:.1f}s\")\n",
|
||||||
|
"\n",
|
||||||
|
"total_params = sum(p.numel() for p in model.parameters())\n",
|
||||||
|
"print(f\"Parameters: {total_params/1e9:.2f}B\")\n",
|
||||||
|
"print(f\"Device: {model.device}, Dtype: {model.dtype}\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" vram_used = torch.cuda.memory_allocated() / 1e9\n",
|
||||||
|
" print(f\"VRAM allocated: {vram_used:.2f} GB\")\n",
|
||||||
|
"ram_used = psutil.Process(os.getpid()).memory_info().rss / 1e9\n",
|
||||||
|
"print(f\"RAM used: {ram_used:.2f} GB\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 3. Text Generation Throughput"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"def throughput(prompt, max_tokens=256, runs=3):\n",
|
||||||
|
" msgs = [{\"role\": \"user\", \"content\": prompt}]\n",
|
||||||
|
" inputs = processor.apply_chat_template(msgs, tokenize=True, return_dict=True,\n",
|
||||||
|
" return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
|
||||||
|
" inp_len = inputs[\"input_ids\"].shape[-1]\n",
|
||||||
|
" lats, toks = [], []\n",
|
||||||
|
" for _ in range(runs):\n",
|
||||||
|
" start = time.perf_counter()\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out = model.generate(**inputs, max_new_tokens=max_tokens, do_sample=True, temperature=0.7)\n",
|
||||||
|
" elapsed = time.perf_counter() - start\n",
|
||||||
|
" gen = out[0][inp_len:]\n",
|
||||||
|
" lats.append(elapsed)\n",
|
||||||
|
" toks.append(len(gen))\n",
|
||||||
|
" tps = [t/l for t,l in zip(toks, lats)]\n",
|
||||||
|
" return {\"prompt\": prompt[:60]+\"...\", \"tokens\": int(np.mean(toks)),\n",
|
||||||
|
" \"latency\": float(np.mean(lats)), \"tps\": float(np.mean(tps))}\n",
|
||||||
|
"\n",
|
||||||
|
"prompts = {\n",
|
||||||
|
" \"Simple QA\": \"What is the capital of Indonesia?\",\n",
|
||||||
|
" \"Math\": \"If a train travels at 120 km/h and another at 80 km/h toward each other from 500 km apart, how long until they meet?\",\n",
|
||||||
|
" \"Code\": \"Write a Python function to find the longest palindromic substring.\",\n",
|
||||||
|
"}\n",
|
||||||
|
"\n",
|
||||||
|
"results = []\n",
|
||||||
|
"for name, p in prompts.items():\n",
|
||||||
|
" r = throughput(p, runs=2)\n",
|
||||||
|
" results.append(r)\n",
|
||||||
|
" print(f\"{name}: {r['tokens']} tok, {r['latency']:.1f}s, {r['tps']:.1f} tok/s\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 4. Reasoning (MMLU-style)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"mmlu = [\n",
|
||||||
|
" {\"q\": \"What is the time complexity of binary search?\", \"o\": [\"A. O(n)\", \"B. O(log n)\", \"C. O(n log n)\", \"D. O(1)\"], \"a\": \"B\"},\n",
|
||||||
|
" {\"q\": \"Which planet has the strongest surface gravity?\", \"o\": [\"A. Earth\", \"B. Mars\", \"C. Jupiter\", \"D. Saturn\"], \"a\": \"C\"},\n",
|
||||||
|
" {\"q\": \"In C++, which keyword prevents overriding?\", \"o\": [\"A. static\", \"B. const\", \"C. final\", \"D. override\"], \"a\": \"C\"},\n",
|
||||||
|
" {\"q\": \"Probability of drawing a red ball from 3 red + 5 blue?\", \"o\": [\"A. 3/5\", \"B. 3/8\", \"C. 5/8\", \"D. 1/2\"], \"a\": \"B\"},\n",
|
||||||
|
" {\"q\": \"What does mitochondria do?\", \"o\": [\"A. Protein\", \"B. Energy (ATP)\", \"C. Lipid\", \"D. DNA\"], \"a\": \"B\"},\n",
|
||||||
|
"]\n",
|
||||||
|
"\n",
|
||||||
|
"ok = 0\n",
|
||||||
|
"for q in mmlu:\n",
|
||||||
|
" prompt = f\"{q['q']}\\n\\n\" + \"\\n\".join(q[\"o\"]) + \"\\n\\nAnswer with a single letter:\"\n",
|
||||||
|
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
|
||||||
|
" tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out = model.generate(**inputs, max_new_tokens=8, do_sample=False)\n",
|
||||||
|
" ans = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n",
|
||||||
|
" cor = q[\"a\"] in ans.upper()[:1]\n",
|
||||||
|
" if cor: ok += 1\n",
|
||||||
|
" print(f\" {'OK' if cor else 'NO'} Expected={q['a']} Got={ans[:20]} | {q['q'][:50]}\")\n",
|
||||||
|
"\n",
|
||||||
|
"print(f\"\\nAccuracy: {ok}/{len(mmlu)} = {ok/len(mmlu)*100:.0f}%\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 5. Coding"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"for name, prompt in [\n",
|
||||||
|
" (\"Binary Search\", \"Write Python `binary_search(arr, target)` returning index or -1.\"),\n",
|
||||||
|
" (\"Fibonacci\", \"Write Python `fib(n)` for nth Fibonacci using DP.\"),\n",
|
||||||
|
"]:\n",
|
||||||
|
" print(f\"\\n{'='*40}\\n{name}\\n{'='*40}\")\n",
|
||||||
|
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
|
||||||
|
" tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out = model.generate(**inputs, max_new_tokens=512, temperature=0.2, do_sample=False)\n",
|
||||||
|
" print(processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)[:400])"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 6. Image Understanding"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"print(\"Test image understanding (16x downsample / efisien)...\")\n",
|
||||||
|
"try:\n",
|
||||||
|
" url = \"https://huggingface.co/datasets/openbmb/DemoCase/resolve/main/refract.png\"\n",
|
||||||
|
" img = Image.open(BytesIO(requests.get(url, timeout=30).content))\n",
|
||||||
|
" display(img.resize((250, 180)))\n",
|
||||||
|
"\n",
|
||||||
|
" msgs = [{\"role\":\"user\",\"content\":[\n",
|
||||||
|
" {\"type\":\"image\",\"image\":img},\n",
|
||||||
|
" {\"type\":\"text\",\"text\":\"What causes this phenomenon?\"}\n",
|
||||||
|
" ]}]\n",
|
||||||
|
"\n",
|
||||||
|
" inputs = processor.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True,\n",
|
||||||
|
" return_dict=True, return_tensors=\"pt\", downsample_mode=DOWNSAMPLE_MODE,\n",
|
||||||
|
" ).to(model.device)\n",
|
||||||
|
"\n",
|
||||||
|
" t0 = time.perf_counter()\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out = model.generate(**inputs, downsample_mode=DOWNSAMPLE_MODE, max_new_tokens=128, do_sample=True)\n",
|
||||||
|
" resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)\n",
|
||||||
|
" print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n",
|
||||||
|
" print(f\"Answer: {resp[:300]}\")\n",
|
||||||
|
"except Exception as e:\n",
|
||||||
|
" print(f\"ERROR: {e}\")\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"\\n---\\nTest image understanding (4x downsample / detail tinggi)...\")\n",
|
||||||
|
"try:\n",
|
||||||
|
" url2 = \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG\"\n",
|
||||||
|
" img2 = Image.open(BytesIO(requests.get(url2, timeout=30).content))\n",
|
||||||
|
" display(img2.resize((250, 180)))\n",
|
||||||
|
"\n",
|
||||||
|
" msgs2 = [{\"role\":\"user\",\"content\":[\n",
|
||||||
|
" {\"type\":\"image\",\"image\":img2},\n",
|
||||||
|
" {\"type\":\"text\",\"text\":\"What animal is on the candy?\"}\n",
|
||||||
|
" ]}]\n",
|
||||||
|
"\n",
|
||||||
|
" inputs2 = processor.apply_chat_template(msgs2, tokenize=True, add_generation_prompt=True,\n",
|
||||||
|
" return_dict=True, return_tensors=\"pt\", downsample_mode=\"4x\",\n",
|
||||||
|
" ).to(model.device)\n",
|
||||||
|
"\n",
|
||||||
|
" t0 = time.perf_counter()\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out2 = model.generate(**inputs2, downsample_mode=\"4x\", max_new_tokens=128, do_sample=True)\n",
|
||||||
|
" resp2 = processor.decode(out2[0][inputs2[\"input_ids\"].shape[-1]:], skip_special_tokens=True)\n",
|
||||||
|
" print(f\"Time: {time.perf_counter()-t0:.1f}s\")\n",
|
||||||
|
" print(f\"Answer: {resp2[:300]}\")\n",
|
||||||
|
"except Exception as e:\n",
|
||||||
|
" print(f\"ERROR: {e}\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 7. Long Context (Needle-in-Haystack)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"def needle(pos):\n",
|
||||||
|
" needle_str = \"The secret code is BLUE-42-GREEN.\"\n",
|
||||||
|
" filler = \"The quick brown fox jumps over the lazy dog. Python is versatile. \"\n",
|
||||||
|
" sents = [filler] * 50\n",
|
||||||
|
" if pos == \"early\": sents.insert(0, needle_str)\n",
|
||||||
|
" elif pos == \"middle\": sents.insert(len(sents)//2, needle_str)\n",
|
||||||
|
" else: sents.append(needle_str)\n",
|
||||||
|
" prompt = f\"Read the text and answer.\\n\\nText: {' '.join(sents)}\\n\\nQ: What is the secret code? Answer with code only.\"\n",
|
||||||
|
" inputs = processor.apply_chat_template([{\"role\":\"user\",\"content\":prompt}],\n",
|
||||||
|
" tokenize=True, return_dict=True, return_tensors=\"pt\", add_generation_prompt=True).to(model.device)\n",
|
||||||
|
" t0 = time.perf_counter()\n",
|
||||||
|
" with torch.no_grad():\n",
|
||||||
|
" out = model.generate(**inputs, max_new_tokens=16, do_sample=False)\n",
|
||||||
|
" resp = processor.decode(out[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n",
|
||||||
|
" return {\"pos\": pos, \"tokens\": inputs[\"input_ids\"].shape[-1], \"resp\": resp[:80],\n",
|
||||||
|
" \"correct\": \"BLUE-42-GREEN\" in resp, \"time\": time.perf_counter()-t0}\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"Needle-in-Haystack:\")\n",
|
||||||
|
"for p in [\"early\", \"middle\", \"late\"]:\n",
|
||||||
|
" r = needle(p)\n",
|
||||||
|
" print(f\"{p.upper()}: {r['tokens']} tok, correct={'OK' if r['correct'] else 'NO'}, {r['time']:.1f}s\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 8. Summary"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"rows = [\n",
|
||||||
|
" [\"Model\", MODEL_ID],\n",
|
||||||
|
" [\"Parameters\", f\"{total_params/1e9:.2f}B\"],\n",
|
||||||
|
" [\"Device\", str(model.device)],\n",
|
||||||
|
" [\"Dtype\", str(model.dtype)],\n",
|
||||||
|
"]\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" rows.append([\"VRAM\", f\"{torch.cuda.memory_allocated()/1e9:.2f} GB\"])\n",
|
||||||
|
"rows.append([\"RAM\", f\"{ram_used:.2f} GB\"])\n",
|
||||||
|
"rows.append([\"Load Time\", f\"{load_time:.1f}s\"])\n",
|
||||||
|
"if results:\n",
|
||||||
|
" rows.append([\"Throughput\", f\"{np.mean([r['tps'] for r in results]):.1f} tok/s\"])\n",
|
||||||
|
"rows.append([\"MMLU\", f\"{ok}/{len(mmlu)} ({ok/len(mmlu)*100:.0f}%)\"])\n",
|
||||||
|
"if 'nh' in dir():\n",
|
||||||
|
" rows.append([\"Needle/Haystack\", f\"{sum(1 for r in nh if r['correct'])}/{len(nh)} correct\"])\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"=\"*60)\n",
|
||||||
|
"print(\" MINICPM-V 4.6 — BENCHMARK SUMMARY\")\n",
|
||||||
|
"print(\"=\"*60)\n",
|
||||||
|
"print(tabulate(rows, headers=[\"Metric\",\"Value\"], tablefmt=\"grid\"))\n",
|
||||||
|
"print(\"=\"*60)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"---\n",
|
||||||
|
"## 9. Cleanup"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"del model, processor\n",
|
||||||
|
"gc.collect()\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" torch.cuda.empty_cache()\n",
|
||||||
|
" print(f\"VRAM: {torch.cuda.memory_allocated()/1e9:.2f} GB\")\n",
|
||||||
|
"print(\"Done.\")"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
Reference in new issue
Block a user