| { |
| "schema_version": "1.0", |
| "benchmark_version": "1.0", |
| "updated": "2026-07-27", |
| "models": [ |
| { |
| "id": "google-gemini-3-1-pro", |
| "name": "Gemini 3.1 Pro", |
| "organization": "Google", |
| "params": "—", |
| "type": "closed", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://deepmind.google/technologies/gemini/", |
| "overall": 61.915, |
| "spatial": 71.72, |
| "spatio_temporal": 64.35, |
| "temporal": 40.74, |
| "semantic": 70.86, |
| "2d_localization": 76.15, |
| "2d_referring_expressions": 67.17, |
| "2d_spatial_pointing": 71.84, |
| "single_object_tracking": 64.35, |
| "temporal_localization": 45.69, |
| "dense_video_captioning": 35.78, |
| "event_verification": 69.83, |
| "video_qa": 71.88 |
| }, |
| { |
| "id": "alibaba-qwen3-vl-32b", |
| "name": "Qwen3-VL-32B-Instruct", |
| "organization": "Alibaba", |
| "params": "32B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct", |
| "overall": 55.04875, |
| "spatial": 70.78, |
| "spatio_temporal": 44.19, |
| "temporal": 39.56, |
| "semantic": 65.67, |
| "2d_localization": 64.33, |
| "2d_referring_expressions": 72.39, |
| "2d_spatial_pointing": 75.62, |
| "single_object_tracking": 44.19, |
| "temporal_localization": 49.71, |
| "dense_video_captioning": 29.4, |
| "event_verification": 60.04, |
| "video_qa": 71.3 |
| }, |
| { |
| "id": "nvidia-cosmos-reason2-8b", |
| "name": "Cosmos-Reason2-8B", |
| "organization": "NVIDIA", |
| "params": "8B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-8B", |
| "overall": 54.1825, |
| "spatial": 73.12, |
| "spatio_temporal": 37.69, |
| "temporal": 39.9, |
| "semantic": 66.02, |
| "2d_localization": 83.88, |
| "2d_referring_expressions": 66.88, |
| "2d_spatial_pointing": 68.6, |
| "single_object_tracking": 37.69, |
| "temporal_localization": 47.3, |
| "dense_video_captioning": 32.5, |
| "event_verification": 64.09, |
| "video_qa": 67.95 |
| }, |
| { |
| "id": "alibaba-qwen3-vl-8b", |
| "name": "Qwen3-VL-8B-Instruct", |
| "organization": "Alibaba", |
| "params": "8B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct", |
| "overall": 50.06875, |
| "spatial": 67.26, |
| "spatio_temporal": 33.12, |
| "temporal": 36.98, |
| "semantic": 62.92, |
| "2d_localization": 59.93, |
| "2d_referring_expressions": 73.29, |
| "2d_spatial_pointing": 68.56, |
| "single_object_tracking": 33.12, |
| "temporal_localization": 44.32, |
| "dense_video_captioning": 29.64, |
| "event_verification": 59.39, |
| "video_qa": 66.44 |
| }, |
| { |
| "id": "alibaba-qwen3-5-vl-27b", |
| "name": "Qwen3.5-27B", |
| "organization": "Alibaba", |
| "params": "27B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-27B-Instruct", |
| "overall": 49.25333333, |
| "spatial": 79.04, |
| "spatio_temporal": 24.4, |
| "temporal": 31.93, |
| "semantic": 61.65, |
| "2d_localization": 85.46, |
| "2d_referring_expressions": 76.35, |
| "2d_spatial_pointing": 75.32, |
| "single_object_tracking": 24.4, |
| "temporal_localization": 36.88, |
| "dense_video_captioning": 26.97, |
| "event_verification": 55.34, |
| "video_qa": 67.95 |
| }, |
| { |
| "id": "alibaba-qwen3-5-vl-9b", |
| "name": "Qwen3.5-9B", |
| "organization": "Alibaba", |
| "params": "9B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-9B-Instruct", |
| "overall": 48.48166667, |
| "spatial": 75.26, |
| "spatio_temporal": 23.52, |
| "temporal": 35.95, |
| "semantic": 59.21, |
| "2d_localization": 76.87, |
| "2d_referring_expressions": 77.16, |
| "2d_spatial_pointing": 71.74, |
| "single_object_tracking": 23.52, |
| "temporal_localization": 42.53, |
| "dense_video_captioning": 29.36, |
| "event_verification": 46.78, |
| "video_qa": 71.63 |
| }, |
| { |
| "id": "google-gemini-3-2-flash-lite", |
| "name": "Gemini 3.1 Flash Lite", |
| "organization": "Google", |
| "params": "—", |
| "type": "closed", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://deepmind.google/technologies/gemini/", |
| "overall": 51.335, |
| "spatial": 62.06, |
| "spatio_temporal": 46.68, |
| "temporal": 35.17, |
| "semantic": 61.43, |
| "2d_localization": 69.88, |
| "2d_referring_expressions": 52.62, |
| "2d_spatial_pointing": 63.68, |
| "single_object_tracking": 46.68, |
| "temporal_localization": 37.72, |
| "dense_video_captioning": 32.62, |
| "event_verification": 54.83, |
| "video_qa": 68.03 |
| }, |
| { |
| "id": "nvidia-cosmos-reason2-32b", |
| "name": "Cosmos-Reason2-32B", |
| "organization": "NVIDIA", |
| "params": "32B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": false, |
| "date_evaluated": "2026-05-27", |
| "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-32B", |
| "overall": 48.55, |
| "spatial": 48.88, |
| "spatio_temporal": 33.12, |
| "temporal": 40.26, |
| "semantic": 71.94, |
| "2d_localization": 22.57, |
| "2d_referring_expressions": 50.04, |
| "2d_spatial_pointing": 74.03, |
| "single_object_tracking": 33.12, |
| "temporal_localization": 50.47, |
| "dense_video_captioning": 30.04, |
| "event_verification": 73.58, |
| "video_qa": 70.29 |
| }, |
| { |
| "id": "nvidia-cosmos3-super", |
| "name": "Cosmos3-Super", |
| "organization": "NVIDIA", |
| "params": "64B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-05-29", |
| "model_url": "https://huggingface.co/nvidia/Cosmos3-Super", |
| "overall": 63.01333333, |
| "spatial": 78.71, |
| "spatio_temporal": 62.25, |
| "temporal": 40.72, |
| "semantic": 70.37, |
| "2d_localization": 86.97, |
| "2d_referring_expressions": 76.23, |
| "2d_spatial_pointing": 72.94, |
| "single_object_tracking": 62.25, |
| "temporal_localization": 51.9, |
| "dense_video_captioning": 29.54, |
| "event_verification": 71.28, |
| "video_qa": 69.46 |
| }, |
| { |
| "id": "nvidia-cosmos3-nano", |
| "name": "Cosmos3-Nano", |
| "organization": "NVIDIA", |
| "params": "16B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-05-29", |
| "model_url": "https://huggingface.co/nvidia/Cosmos3-Nano", |
| "overall": 60.67291667, |
| "spatial": 74.84, |
| "spatio_temporal": 59.21, |
| "temporal": 39.73, |
| "semantic": 68.92, |
| "2d_localization": 74.11, |
| "2d_referring_expressions": 75.57, |
| "2d_spatial_pointing": 74.83, |
| "single_object_tracking": 59.21, |
| "temporal_localization": 48.04, |
| "dense_video_captioning": 31.42, |
| "event_verification": 68.88, |
| "video_qa": 68.95 |
| }, |
| { |
| "id": "alibaba-qwen3-vl-2b", |
| "name": "Qwen3-VL-2B-Instruct", |
| "organization": "Alibaba", |
| "params": "2B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-07-16", |
| "model_url": "https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct", |
| "overall": 40.24583333, |
| "spatial": 58.28, |
| "spatio_temporal": 30.38, |
| "temporal": 18.03, |
| "semantic": 54.29, |
| "2d_localization": 56.6, |
| "2d_referring_expressions": 65.12, |
| "2d_spatial_pointing": 53.13, |
| "single_object_tracking": 30.38, |
| "temporal_localization": 35.21, |
| "dense_video_captioning": 0.85, |
| "event_verification": 44.73, |
| "video_qa": 63.85 |
| }, |
| { |
| "id": "nvidia-cosmos-reason2-2b", |
| "name": "Cosmos-Reason2-2B", |
| "organization": "NVIDIA", |
| "params": "2B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-07-16", |
| "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-2B", |
| "overall": 45.84958333, |
| "spatial": 62.83, |
| "spatio_temporal": 26.85, |
| "temporal": 33.74, |
| "semantic": 59.98, |
| "2d_localization": 72.47, |
| "2d_referring_expressions": 56.33, |
| "2d_spatial_pointing": 59.7, |
| "single_object_tracking": 26.85, |
| "temporal_localization": 38.95, |
| "dense_video_captioning": 28.52, |
| "event_verification": 55.27, |
| "video_qa": 64.69 |
| }, |
| { |
| "id": "nvidia-cosmos3-edge", |
| "name": "Cosmos3-Edge", |
| "organization": "NVIDIA", |
| "params": "4B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-07-16", |
| "model_url": "https://huggingface.co/nvidia/Cosmos3-Edge", |
| "overall": 46.93291667, |
| "spatial": 71.99, |
| "spatio_temporal": 24.65, |
| "temporal": 28.08, |
| "semantic": 63.02, |
| "2d_localization": 78.93, |
| "2d_referring_expressions": 69.27, |
| "2d_spatial_pointing": 67.76, |
| "single_object_tracking": 24.65, |
| "temporal_localization": 39.51, |
| "dense_video_captioning": 16.64, |
| "event_verification": 63.36, |
| "video_qa": 62.68 |
| }, |
| { |
| "id": "google-gemma-4-e2b-it", |
| "name": "Gemma-4-E2B-it", |
| "organization": "Google", |
| "params": "2B", |
| "type": "open", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-07-16", |
| "model_url": "https://huggingface.co/google/gemma-4-e2b-it", |
| "overall": 21.55625, |
| "spatial": 28.56, |
| "spatio_temporal": 11.58, |
| "temporal": 9.16, |
| "semantic": 36.92, |
| "2d_localization": 48.42, |
| "2d_referring_expressions": 5.52, |
| "2d_spatial_pointing": 31.74, |
| "single_object_tracking": 11.58, |
| "temporal_localization": 9.02, |
| "dense_video_captioning": 9.31, |
| "event_verification": 27.65, |
| "video_qa": 46.19 |
| }, |
| { |
| "id": "google-gemini-3-6-flash", |
| "name": "Gemini 3.6 Flash", |
| "organization": "Google", |
| "params": "—", |
| "type": "closed", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-07-21", |
| "model_url": "https://deepmind.google/technologies/gemini/", |
| "overall": 69.5187, |
| "spatial": 78.85, |
| "spatio_temporal": 75.99, |
| "temporal": 43.825, |
| "semantic": 79.41, |
| "2d_localization": 81.57, |
| "2d_referring_expressions": 75.78, |
| "2d_spatial_pointing": 79.2, |
| "single_object_tracking": 75.99, |
| "temporal_localization": 51.5, |
| "dense_video_captioning": 36.15, |
| "event_verification": 82, |
| "video_qa": 76.82 |
| }, |
| { |
| "id": "google-gemini-3-5-flash-lite", |
| "name": "Gemini 3.5 Flash-Lite", |
| "organization": "Google", |
| "params": "—", |
| "type": "closed", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-07-21", |
| "model_url": "https://deepmind.google/models/gemini/flash-lite/", |
| "overall": 52.8925, |
| "spatial": 61.09, |
| "spatio_temporal": 55.77, |
| "temporal": 26.7, |
| "semantic": 68.01, |
| "2d_localization": 65.73, |
| "2d_referring_expressions": 74.46, |
| "2d_spatial_pointing": 43.08, |
| "single_object_tracking": 55.77, |
| "temporal_localization": 24, |
| "dense_video_captioning": 29.4, |
| "event_verification": 70.41, |
| "video_qa": 65.61 |
| }, |
| { |
| "id": "openai-gpt-5-6-sol", |
| "name": "GPT-5.6 Sol", |
| "organization": "OpenAI", |
| "params": "—", |
| "type": "closed", |
| "result_type": "single", |
| "verified": true, |
| "ensemble": false, |
| "is_new": true, |
| "date_evaluated": "2026-07-09", |
| "model_url": "https://openai.com/index/gpt-5-6/", |
| "overall": 69.0258, |
| "spatial": 77.9333, |
| "spatio_temporal": 75.02, |
| "temporal": 46.25, |
| "semantic": 76.9, |
| "2d_localization": 69.3, |
| "2d_referring_expressions": 82.6, |
| "2d_spatial_pointing": 81.9, |
| "single_object_tracking": 75.02, |
| "temporal_localization": 54.8, |
| "dense_video_captioning": 37.7, |
| "event_verification": 75.9, |
| "video_qa": 77.9 |
| } |
| ] |
| } |
|
|
