{
  "schema_version": "1.0",
  "benchmark_version": "1.0",
  "updated": "2026-07-27",
  "models": [
    {
      "id": "google-gemini-3-1-pro",
      "name": "Gemini 3.1 Pro",
      "organization": "Google",
      "params": "—",
      "type": "closed",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://deepmind.google/technologies/gemini/",
      "overall": 61.915,
      "spatial": 71.72,
      "spatio_temporal": 64.35,
      "temporal": 40.74,
      "semantic": 70.86,
      "2d_localization": 76.15,
      "2d_referring_expressions": 67.17,
      "2d_spatial_pointing": 71.84,
      "single_object_tracking": 64.35,
      "temporal_localization": 45.69,
      "dense_video_captioning": 35.78,
      "event_verification": 69.83,
      "video_qa": 71.88
    },
    {
      "id": "alibaba-qwen3-vl-32b",
      "name": "Qwen3-VL-32B-Instruct",
      "organization": "Alibaba",
      "params": "32B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct",
      "overall": 55.04875,
      "spatial": 70.78,
      "spatio_temporal": 44.19,
      "temporal": 39.56,
      "semantic": 65.67,
      "2d_localization": 64.33,
      "2d_referring_expressions": 72.39,
      "2d_spatial_pointing": 75.62,
      "single_object_tracking": 44.19,
      "temporal_localization": 49.71,
      "dense_video_captioning": 29.4,
      "event_verification": 60.04,
      "video_qa": 71.3
    },
    {
      "id": "nvidia-cosmos-reason2-8b",
      "name": "Cosmos-Reason2-8B",
      "organization": "NVIDIA",
      "params": "8B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-8B",
      "overall": 54.1825,
      "spatial": 73.12,
      "spatio_temporal": 37.69,
      "temporal": 39.9,
      "semantic": 66.02,
      "2d_localization": 83.88,
      "2d_referring_expressions": 66.88,
      "2d_spatial_pointing": 68.6,
      "single_object_tracking": 37.69,
      "temporal_localization": 47.3,
      "dense_video_captioning": 32.5,
      "event_verification": 64.09,
      "video_qa": 67.95
    },
    {
      "id": "alibaba-qwen3-vl-8b",
      "name": "Qwen3-VL-8B-Instruct",
      "organization": "Alibaba",
      "params": "8B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct",
      "overall": 50.06875,
      "spatial": 67.26,
      "spatio_temporal": 33.12,
      "temporal": 36.98,
      "semantic": 62.92,
      "2d_localization": 59.93,
      "2d_referring_expressions": 73.29,
      "2d_spatial_pointing": 68.56,
      "single_object_tracking": 33.12,
      "temporal_localization": 44.32,
      "dense_video_captioning": 29.64,
      "event_verification": 59.39,
      "video_qa": 66.44
    },
    {
      "id": "alibaba-qwen3-5-vl-27b",
      "name": "Qwen3.5-27B",
      "organization": "Alibaba",
      "params": "27B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-27B-Instruct",
      "overall": 49.25333333,
      "spatial": 79.04,
      "spatio_temporal": 24.4,
      "temporal": 31.93,
      "semantic": 61.65,
      "2d_localization": 85.46,
      "2d_referring_expressions": 76.35,
      "2d_spatial_pointing": 75.32,
      "single_object_tracking": 24.4,
      "temporal_localization": 36.88,
      "dense_video_captioning": 26.97,
      "event_verification": 55.34,
      "video_qa": 67.95
    },
    {
      "id": "alibaba-qwen3-5-vl-9b",
      "name": "Qwen3.5-9B",
      "organization": "Alibaba",
      "params": "9B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-9B-Instruct",
      "overall": 48.48166667,
      "spatial": 75.26,
      "spatio_temporal": 23.52,
      "temporal": 35.95,
      "semantic": 59.21,
      "2d_localization": 76.87,
      "2d_referring_expressions": 77.16,
      "2d_spatial_pointing": 71.74,
      "single_object_tracking": 23.52,
      "temporal_localization": 42.53,
      "dense_video_captioning": 29.36,
      "event_verification": 46.78,
      "video_qa": 71.63
    },
    {
      "id": "google-gemini-3-2-flash-lite",
      "name": "Gemini 3.1 Flash Lite",
      "organization": "Google",
      "params": "—",
      "type": "closed",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://deepmind.google/technologies/gemini/",
      "overall": 51.335,
      "spatial": 62.06,
      "spatio_temporal": 46.68,
      "temporal": 35.17,
      "semantic": 61.43,
      "2d_localization": 69.88,
      "2d_referring_expressions": 52.62,
      "2d_spatial_pointing": 63.68,
      "single_object_tracking": 46.68,
      "temporal_localization": 37.72,
      "dense_video_captioning": 32.62,
      "event_verification": 54.83,
      "video_qa": 68.03
    },
    {
      "id": "nvidia-cosmos-reason2-32b",
      "name": "Cosmos-Reason2-32B",
      "organization": "NVIDIA",
      "params": "32B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": false,
      "date_evaluated": "2026-05-27",
      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-32B",
      "overall": 48.55,
      "spatial": 48.88,
      "spatio_temporal": 33.12,
      "temporal": 40.26,
      "semantic": 71.94,
      "2d_localization": 22.57,
      "2d_referring_expressions": 50.04,
      "2d_spatial_pointing": 74.03,
      "single_object_tracking": 33.12,
      "temporal_localization": 50.47,
      "dense_video_captioning": 30.04,
      "event_verification": 73.58,
      "video_qa": 70.29
    },
    {
      "id": "nvidia-cosmos3-super",
      "name": "Cosmos3-Super",
      "organization": "NVIDIA",
      "params": "64B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-05-29",
      "model_url": "https://huggingface.co/nvidia/Cosmos3-Super",
      "overall": 63.01333333,
      "spatial": 78.71,
      "spatio_temporal": 62.25,
      "temporal": 40.72,
      "semantic": 70.37,
      "2d_localization": 86.97,
      "2d_referring_expressions": 76.23,
      "2d_spatial_pointing": 72.94,
      "single_object_tracking": 62.25,
      "temporal_localization": 51.9,
      "dense_video_captioning": 29.54,
      "event_verification": 71.28,
      "video_qa": 69.46
    },
    {
      "id": "nvidia-cosmos3-nano",
      "name": "Cosmos3-Nano",
      "organization": "NVIDIA",
      "params": "16B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-05-29",
      "model_url": "https://huggingface.co/nvidia/Cosmos3-Nano",
      "overall": 60.67291667,
      "spatial": 74.84,
      "spatio_temporal": 59.21,
      "temporal": 39.73,
      "semantic": 68.92,
      "2d_localization": 74.11,
      "2d_referring_expressions": 75.57,
      "2d_spatial_pointing": 74.83,
      "single_object_tracking": 59.21,
      "temporal_localization": 48.04,
      "dense_video_captioning": 31.42,
      "event_verification": 68.88,
      "video_qa": 68.95
    },
    {
      "id": "alibaba-qwen3-vl-2b",
      "name": "Qwen3-VL-2B-Instruct",
      "organization": "Alibaba",
      "params": "2B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-07-16",
      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct",
      "overall": 40.24583333,
      "spatial": 58.28,
      "spatio_temporal": 30.38,
      "temporal": 18.03,
      "semantic": 54.29,
      "2d_localization": 56.6,
      "2d_referring_expressions": 65.12,
      "2d_spatial_pointing": 53.13,
      "single_object_tracking": 30.38,
      "temporal_localization": 35.21,
      "dense_video_captioning": 0.85,
      "event_verification": 44.73,
      "video_qa": 63.85
    },
    {
      "id": "nvidia-cosmos-reason2-2b",
      "name": "Cosmos-Reason2-2B",
      "organization": "NVIDIA",
      "params": "2B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-07-16",
      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-2B",
      "overall": 45.84958333,
      "spatial": 62.83,
      "spatio_temporal": 26.85,
      "temporal": 33.74,
      "semantic": 59.98,
      "2d_localization": 72.47,
      "2d_referring_expressions": 56.33,
      "2d_spatial_pointing": 59.7,
      "single_object_tracking": 26.85,
      "temporal_localization": 38.95,
      "dense_video_captioning": 28.52,
      "event_verification": 55.27,
      "video_qa": 64.69
    },
    {
      "id": "nvidia-cosmos3-edge",
      "name": "Cosmos3-Edge",
      "organization": "NVIDIA",
      "params": "4B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-07-16",
      "model_url": "https://huggingface.co/nvidia/Cosmos3-Edge",
      "overall": 46.93291667,
      "spatial": 71.99,
      "spatio_temporal": 24.65,
      "temporal": 28.08,
      "semantic": 63.02,
      "2d_localization": 78.93,
      "2d_referring_expressions": 69.27,
      "2d_spatial_pointing": 67.76,
      "single_object_tracking": 24.65,
      "temporal_localization": 39.51,
      "dense_video_captioning": 16.64,
      "event_verification": 63.36,
      "video_qa": 62.68
    },
    {
      "id": "google-gemma-4-e2b-it",
      "name": "Gemma-4-E2B-it",
      "organization": "Google",
      "params": "2B",
      "type": "open",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-07-16",
      "model_url": "https://huggingface.co/google/gemma-4-e2b-it",
      "overall": 21.55625,
      "spatial": 28.56,
      "spatio_temporal": 11.58,
      "temporal": 9.16,
      "semantic": 36.92,
      "2d_localization": 48.42,
      "2d_referring_expressions": 5.52,
      "2d_spatial_pointing": 31.74,
      "single_object_tracking": 11.58,
      "temporal_localization": 9.02,
      "dense_video_captioning": 9.31,
      "event_verification": 27.65,
      "video_qa": 46.19
    },
    {
      "id": "google-gemini-3-6-flash",
      "name": "Gemini 3.6 Flash",
      "organization": "Google",
      "params": "—",
      "type": "closed",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-07-21",
      "model_url": "https://deepmind.google/technologies/gemini/",
      "overall": 69.5187,
      "spatial": 78.85,
      "spatio_temporal": 75.99,
      "temporal": 43.825,
      "semantic": 79.41,
      "2d_localization": 81.57,
      "2d_referring_expressions": 75.78,
      "2d_spatial_pointing": 79.2,
      "single_object_tracking": 75.99,
      "temporal_localization": 51.5,
      "dense_video_captioning": 36.15,
      "event_verification": 82,
      "video_qa": 76.82
    },
    {
      "id": "google-gemini-3-5-flash-lite",
      "name": "Gemini 3.5 Flash-Lite",
      "organization": "Google",
      "params": "—",
      "type": "closed",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-07-21",
      "model_url": "https://deepmind.google/models/gemini/flash-lite/",
      "overall": 52.8925,
      "spatial": 61.09,
      "spatio_temporal": 55.77,
      "temporal": 26.7,
      "semantic": 68.01,
      "2d_localization": 65.73,
      "2d_referring_expressions": 74.46,
      "2d_spatial_pointing": 43.08,
      "single_object_tracking": 55.77,
      "temporal_localization": 24,
      "dense_video_captioning": 29.4,
      "event_verification": 70.41,
      "video_qa": 65.61
    },
    {
      "id": "openai-gpt-5-6-sol",
      "name": "GPT-5.6 Sol",
      "organization": "OpenAI",
      "params": "—",
      "type": "closed",
      "result_type": "single",
      "verified": true,
      "ensemble": false,
      "is_new": true,
      "date_evaluated": "2026-07-09",
      "model_url": "https://openai.com/index/gpt-5-6/",
      "overall": 69.0258,
      "spatial": 77.9333,
      "spatio_temporal": 75.02,
      "temporal": 46.25,
      "semantic": 76.9,
      "2d_localization": 69.3,
      "2d_referring_expressions": 82.6,
      "2d_spatial_pointing": 81.9,
      "single_object_tracking": 75.02,
      "temporal_localization": 54.8,
      "dense_video_captioning": 37.7,
      "event_verification": 75.9,
      "video_qa": 77.9
    }
  ]
}