{
  "suite": "swebench-verified",
  "subset": "subset-50-seed20260804",
  "run_date": "2026-08-09",
  "model": "plori-auto",
  "concurrency": 8,
  "publication": {
    "system": "forge + Plori Router",
    "platform_release": "control-plane v135",
    "router": "plori-auto",
    "router_pool": "paid",
    "harness_version": "swebench 4.1.0",
    "execution_mode": "patch-only",
    "run_type": "single",
    "repeats": 1,
    "task_deadline_seconds": 1500,
    "environment_note": "The run started with six pre-warmed agent nodes and held eight nodes for eight concurrent tasks, one task per node.",
    "variance_note": "Single run. Treat 52% as low-50s rather than an exact capability estimate; no repeated-run variance is available for this configuration.",
    "reproduction_url": "https://github.com/liu1700/plori-runtime/tree/ae1997af9ec6b685f8287e4bec82fc438da8faa9/bench/results/2026-08-09/swebench-50-c8",
    "source_commit": "ae1997af9ec6b685f8287e4bec82fc438da8faa9",
    "limitations": [
      {
        "id": "incomplete-checkout",
        "title": "Four tasks were affected by an incomplete checkout",
        "summary": "One retained patch proves the repository checkout stopped before its alphabetical tail; three unretrievable oversized patches share the same signature. The published 52% is the result this run produced, not a measurement of the agent's ceiling on those four tasks.",
        "task_ids": [
          "django__django-9296",
          "sphinx-doc__sphinx-11510",
          "django__django-12663",
          "django__django-11179"
        ]
      },
      {
        "id": "harness-error",
        "title": "One submitted patch errored in the official harness",
        "summary": "django__django-11179 was submitted but not scored. Harness errors are reported separately from unresolved patches.",
        "task_ids": [
          "django__django-11179"
        ]
      }
    ]
  },
  "totals": {
    "instances": 50,
    "scored": 29,
    "resolved": 26,
    "resolve_rate": 0.52,
    "model_cost_usd": 20.5833,
    "infra_cost_usd": 7.957,
    "total_cost_usd": 28.5404,
    "infra_cost_share": 0.2788,
    "model_distribution": [
      {
        "model": "deepseek/deepseek-v4-flash-0731",
        "calls": 2016,
        "tokens": 31389278,
        "credits": 2016,
        "share": 0.8682
      },
      {
        "model": "openai/gpt-5.6-luna",
        "calls": 224,
        "tokens": 3928797,
        "credits": 224,
        "share": 0.0965
      },
      {
        "model": "moonshotai/kimi-k3",
        "calls": 82,
        "tokens": 1014637,
        "credits": 230,
        "share": 0.0353
      }
    ],
    "wall_seconds_sum": 57290.62113194398,
    "wall_seconds_suite": 7905.496844238,
    "tokens_total": 36332712,
    "patches_salvaged": 4
  },
  "latency": {
    "time_to_first_event_p50_seconds": 6.137586,
    "time_to_first_event_p90_seconds": 9.66691,
    "time_to_first_tool_p50_seconds": 15.501867,
    "wall_seconds_p50": 1121.308682,
    "create_agent_p50_ms": 147,
    "submit_run_p50_ms": 442,
    "tool_calls_p50": 53,
    "samples": 50
  },
  "tasks": [
    {
      "instance_id": "astropy__astropy-13236",
      "resolved": false,
      "scored": false,
      "tokens": 166401,
      "credits": 44,
      "model_cost_usd": 0.1917,
      "infra_cost_usd": 0.1744,
      "total_cost_usd": 0.366,
      "wall_seconds": 1255.491596,
      "time_to_first_event_seconds": 7.448357,
      "tool_calls": 25,
      "approvals_granted": 1,
      "failure_reason": "run_error",
      "run_ids": [
        "7c16ea69d4a90fb5830fcdc1ab72a47f",
        "b7ca02535e002283a208e6f2f07dfc1b",
        "bfdb1fc11936ab26926c74d527f515d4"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 23,
          "tokens": 166401,
          "credits": 23
        }
      ]
    },
    {
      "instance_id": "astropy__astropy-13398",
      "resolved": true,
      "scored": true,
      "tokens": 1135701,
      "credits": 63,
      "model_cost_usd": 0.375,
      "infra_cost_usd": 0.105,
      "total_cost_usd": 0.48,
      "wall_seconds": 755.69751,
      "time_to_first_event_seconds": 6.858107,
      "tool_calls": 59,
      "approvals_granted": 1,
      "run_ids": [
        "47af64d18f543d3f06ed79f86a104b26",
        "6bc3ddc480235722cef6120b09a7e6fc"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 45,
          "tokens": 1135701,
          "credits": 45
        }
      ]
    },
    {
      "instance_id": "astropy__astropy-13453",
      "resolved": false,
      "scored": false,
      "tokens": 1059411,
      "credits": 68,
      "model_cost_usd": 0.425,
      "infra_cost_usd": 0.0902,
      "total_cost_usd": 0.5152,
      "wall_seconds": 649.212023,
      "time_to_first_event_seconds": 6.355372,
      "tool_calls": 52,
      "approvals_granted": 0,
      "failure_reason": "patch_empty",
      "run_ids": [
        "b99a313eeda9ce8877eeb6baf7dca2b5",
        "ad91e37280e028068e153f1904611cf6"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 51,
          "tokens": 1059411,
          "credits": 51
        }
      ]
    },
    {
      "instance_id": "django__django-10554",
      "resolved": false,
      "scored": false,
      "tokens": 118807,
      "credits": 30,
      "model_cost_usd": 0.1417,
      "infra_cost_usd": 0.0916,
      "total_cost_usd": 0.2333,
      "wall_seconds": 659.47053,
      "time_to_first_event_seconds": 5.491576,
      "tool_calls": 18,
      "approvals_granted": 1,
      "failure_reason": "patch_empty",
      "run_ids": [
        "b196e78f84a8c70315acc515025e93df",
        "fd1cd30f1ba7ea2c22ba88e9564a88b0",
        "8fc237861a725de80546330a0552f100"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 17,
          "tokens": 118807,
          "credits": 17
        }
      ]
    },
    {
      "instance_id": "django__django-10880",
      "resolved": true,
      "scored": true,
      "tokens": 253684,
      "credits": 37,
      "model_cost_usd": 0.2083,
      "infra_cost_usd": 0.0994,
      "total_cost_usd": 0.3078,
      "wall_seconds": 715.817238,
      "time_to_first_event_seconds": 5.783952,
      "tool_calls": 29,
      "approvals_granted": 0,
      "run_ids": [
        "bba7e7af63112e4f18b9cad487a90598"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 25,
          "tokens": 253684,
          "credits": 25
        }
      ]
    },
    {
      "instance_id": "django__django-11066",
      "resolved": true,
      "scored": true,
      "tokens": 260640,
      "credits": 91,
      "model_cost_usd": 0.6083,
      "infra_cost_usd": 0.1463,
      "total_cost_usd": 0.7547,
      "wall_seconds": 1053.625073,
      "time_to_first_event_seconds": 5.460686,
      "tool_calls": 27,
      "approvals_granted": 0,
      "run_ids": [
        "c7995ac867a413cf3d0e199c91335f01"
      ],
      "models": [
        {
          "model": "moonshotai/kimi-k3",
          "calls": 24,
          "tokens": 237725,
          "credits": 69
        },
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 4,
          "tokens": 22915,
          "credits": 4
        }
      ]
    },
    {
      "instance_id": "django__django-11179",
      "resolved": false,
      "scored": false,
      "tokens": 119087,
      "credits": 52,
      "model_cost_usd": 0.1583,
      "infra_cost_usd": 0.2499,
      "total_cost_usd": 0.4083,
      "wall_seconds": 1799.627453646,
      "time_to_first_event_seconds": 5.256951,
      "tool_calls": 17,
      "approvals_granted": 0,
      "salvaged": true,
      "failure_reason": "timeout",
      "run_ids": [
        "55cebdfb13e2fd1c92a26d9d7d50bcd7",
        "68fd199b278f8121b3cd9c700e559b3d"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 12,
          "tokens": 72120,
          "credits": 12
        },
        {
          "model": "openai/gpt-5.6-luna",
          "calls": 7,
          "tokens": 46967,
          "credits": 7
        }
      ]
    },
    {
      "instance_id": "django__django-11490",
      "resolved": true,
      "scored": true,
      "tokens": 1030464,
      "credits": 84,
      "model_cost_usd": 0.4917,
      "infra_cost_usd": 0.1971,
      "total_cost_usd": 0.6888,
      "wall_seconds": 1419.286765,
      "time_to_first_event_seconds": 6.3226189999999995,
      "tool_calls": 64,
      "approvals_granted": 0,
      "run_ids": [
        "f0da163d9e7025770f624c2f0263df3e"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 59,
          "tokens": 1030464,
          "credits": 59
        }
      ]
    },
    {
      "instance_id": "django__django-11964",
      "resolved": true,
      "scored": true,
      "tokens": 726643,
      "credits": 73,
      "model_cost_usd": 0.4333,
      "infra_cost_usd": 0.1557,
      "total_cost_usd": 0.5891,
      "wall_seconds": 1121.308682,
      "time_to_first_event_seconds": 5.224944,
      "tool_calls": 56,
      "approvals_granted": 2,
      "run_ids": [
        "cb38a26e7ca704456f4506f06447724e",
        "afc5a0d0b34d99ba6000b6c63641270c",
        "b1b62780327c9d8ccce84c9ffa4beda1"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 52,
          "tokens": 726643,
          "credits": 52
        }
      ]
    },
    {
      "instance_id": "django__django-11999",
      "resolved": true,
      "scored": true,
      "tokens": 961189,
      "credits": 92,
      "model_cost_usd": 0.5583,
      "infra_cost_usd": 0.2154,
      "total_cost_usd": 0.7737,
      "wall_seconds": 1550.787894,
      "time_to_first_event_seconds": 5.416224,
      "tool_calls": 68,
      "approvals_granted": 5,
      "salvaged": true,
      "failure_reason": "run_error",
      "run_ids": [
        "8597fe1488eb5bebbaeaad9e3f7b5377",
        "0cabbfa8802aa62672c8c50514b45905",
        "bf566d30413a6b3de46851168e01362a",
        "8569843ead04013492bd21b96103e32d",
        "7e1e15b04727d085f8b61a4c0d4c9295",
        "3358875e2d662f062ade2cb0757eb648"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 67,
          "tokens": 961189,
          "credits": 67
        }
      ]
    },
    {
      "instance_id": "django__django-12155",
      "resolved": true,
      "scored": true,
      "tokens": 758286,
      "credits": 59,
      "model_cost_usd": 0.4,
      "infra_cost_usd": 0.0763,
      "total_cost_usd": 0.4763,
      "wall_seconds": 549.07898,
      "time_to_first_event_seconds": 4.704268,
      "tool_calls": 52,
      "approvals_granted": 0,
      "run_ids": [
        "94e6c2bc6581342aa1f54b42ff7d8c05"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 48,
          "tokens": 758286,
          "credits": 48
        }
      ]
    },
    {
      "instance_id": "django__django-12193",
      "resolved": true,
      "scored": true,
      "tokens": 607447,
      "credits": 61,
      "model_cost_usd": 0.4,
      "infra_cost_usd": 0.0946,
      "total_cost_usd": 0.4946,
      "wall_seconds": 681.46896,
      "time_to_first_event_seconds": 6.080665,
      "tool_calls": 53,
      "approvals_granted": 0,
      "run_ids": [
        "8b131b34e143aaef78dc497a012bba22"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 48,
          "tokens": 607447,
          "credits": 48
        }
      ]
    },
    {
      "instance_id": "django__django-12209",
      "resolved": true,
      "scored": true,
      "tokens": 364141,
      "credits": 47,
      "model_cost_usd": 0.25,
      "infra_cost_usd": 0.1316,
      "total_cost_usd": 0.3816,
      "wall_seconds": 947.473148,
      "time_to_first_event_seconds": 6.577035,
      "tool_calls": 32,
      "approvals_granted": 0,
      "run_ids": [
        "18714568d06e6bbed8e64783e7d403cd"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 30,
          "tokens": 364141,
          "credits": 30
        }
      ]
    },
    {
      "instance_id": "django__django-12273",
      "resolved": false,
      "scored": false,
      "tokens": 407615,
      "credits": 66,
      "model_cost_usd": 0.275,
      "infra_cost_usd": 0.25,
      "total_cost_usd": 0.525,
      "wall_seconds": 1799.739024675,
      "time_to_first_event_seconds": 6.486483,
      "tool_calls": 40,
      "approvals_granted": 0,
      "failure_reason": "timeout",
      "run_ids": [
        "1077062d33b859e254a136304f5eb38e",
        "33ce03f18c4a99dbf6fd743028e7f11f"
      ],
      "models": [
        {
          "model": "openai/gpt-5.6-luna",
          "calls": 31,
          "tokens": 398521,
          "credits": 31
        },
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 2,
          "tokens": 9094,
          "credits": 2
        }
      ]
    },
    {
      "instance_id": "django__django-12663",
      "resolved": false,
      "scored": false,
      "tokens": 640370,
      "credits": 83,
      "model_cost_usd": 0.4,
      "infra_cost_usd": 0.2499,
      "total_cost_usd": 0.6499,
      "wall_seconds": 1799.547266146,
      "time_to_first_event_seconds": 5.941912,
      "tool_calls": 54,
      "approvals_granted": 4,
      "failure_reason": "timeout",
      "run_ids": [
        "9a4565b6cd1060f4e5f43d6b7126d4b2",
        "f0fc6f0f1679b16d98359f0f43aa288d",
        "340793cc183ebdcd79050b8214f0d26d",
        "8249f798fa16e68a69aab608f0d8e5bf",
        "a5f8b12dc40e734faa4f39e90140b1b7",
        "7fab67cab499fbe8bd76f9fcb0ba5866"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 48,
          "tokens": 640370,
          "credits": 48
        }
      ]
    },
    {
      "instance_id": "django__django-12754",
      "resolved": true,
      "scored": true,
      "tokens": 925673,
      "credits": 62,
      "model_cost_usd": 0.4833,
      "infra_cost_usd": 0.2499,
      "total_cost_usd": 0.7332,
      "wall_seconds": 1799.346385642,
      "time_to_first_event_seconds": 4.671441,
      "tool_calls": 62,
      "approvals_granted": 0,
      "salvaged": true,
      "failure_reason": "timeout",
      "run_ids": [
        "74011bd3b41783bba1f45e4e195a9360"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 58,
          "tokens": 925673,
          "credits": 58
        }
      ]
    },
    {
      "instance_id": "django__django-13028",
      "resolved": false,
      "scored": false,
      "tokens": 86712,
      "credits": 16,
      "model_cost_usd": 0.125,
      "infra_cost_usd": 0.0514,
      "total_cost_usd": 0.1764,
      "wall_seconds": 370.158143,
      "time_to_first_event_seconds": 8.700795,
      "tool_calls": 13,
      "approvals_granted": 0,
      "failure_reason": "run_error",
      "run_ids": [
        "3dc288e9ab5e378293ae700a93ec8b5c",
        "cc56a0874b8ce5814f082ad909e43798"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 13,
          "tokens": 81504,
          "credits": 13
        },
        {
          "model": "openai/gpt-5.6-luna",
          "calls": 2,
          "tokens": 5208,
          "credits": 2
        }
      ]
    },
    {
      "instance_id": "django__django-13112",
      "resolved": false,
      "scored": false,
      "tokens": 1059201,
      "credits": 95,
      "model_cost_usd": 0.625,
      "infra_cost_usd": 0.1519,
      "total_cost_usd": 0.7769,
      "wall_seconds": 1093.770104,
      "time_to_first_event_seconds": 7.062897,
      "tool_calls": 72,
      "approvals_granted": 0,
      "failure_reason": "patch_empty",
      "run_ids": [
        "a0d18222f25ac155c97978cdd257a781",
        "78efb05f205acb9264969d80c7eb5599"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 75,
          "tokens": 1059201,
          "credits": 75
        }
      ]
    },
    {
      "instance_id": "django__django-13297",
      "resolved": false,
      "scored": false,
      "tokens": 1050347,
      "credits": 85,
      "model_cost_usd": 0.5083,
      "infra_cost_usd": 0.1863,
      "total_cost_usd": 0.6947,
      "wall_seconds": 1341.669959,
      "time_to_first_event_seconds": 7.782336,
      "tool_calls": 62,
      "approvals_granted": 0,
      "failure_reason": "patch_empty",
      "run_ids": [
        "6acbb0f9fd5e8d39427e45779b452dfd",
        "a01f9d3c3f56b90e81533ebd4a74f9c0"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 61,
          "tokens": 1050347,
          "credits": 61
        }
      ]
    },
    {
      "instance_id": "django__django-13344",
      "resolved": false,
      "scored": false,
      "tokens": 255937,
      "credits": 52,
      "model_cost_usd": 0.2667,
      "infra_cost_usd": 0.1502,
      "total_cost_usd": 0.4168,
      "wall_seconds": 1081.29943,
      "time_to_first_event_seconds": 6.453121,
      "tool_calls": 30,
      "approvals_granted": 0,
      "failure_reason": "patch_empty",
      "run_ids": [
        "5d3da064b9c76e2ddf6571012e60f78f",
        "4a80df439267eea694e01066a250d48d"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 32,
          "tokens": 255937,
          "credits": 32
        }
      ]
    },
    {
      "instance_id": "django__django-13820",
      "resolved": true,
      "scored": true,
      "tokens": 414956,
      "credits": 41,
      "model_cost_usd": 0.2417,
      "infra_cost_usd": 0.069,
      "total_cost_usd": 0.3107,
      "wall_seconds": 496.980492,
      "time_to_first_event_seconds": 6.995861,
      "tool_calls": 34,
      "approvals_granted": 0,
      "run_ids": [
        "cd812418676bf5ed37163b7b2b46f114"
      ],
      "models": [
        {
          "model": "openai/gpt-5.6-luna",
          "calls": 27,
          "tokens": 405337,
          "credits": 27
        },
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 2,
          "tokens": 9619,
          "credits": 2
        }
      ]
    },
    {
      "instance_id": "django__django-14034",
      "resolved": true,
      "scored": true,
      "tokens": 445583,
      "credits": 53,
      "model_cost_usd": 0.3333,
      "infra_cost_usd": 0.0986,
      "total_cost_usd": 0.432,
      "wall_seconds": 710.253343,
      "time_to_first_event_seconds": 9.66691,
      "tool_calls": 39,
      "approvals_granted": 0,
      "run_ids": [
        "43c4900106592879e888d4605375682f"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 40,
          "tokens": 445583,
          "credits": 40
        }
      ]
    },
    {
      "instance_id": "django__django-14155",
      "resolved": false,
      "scored": false,
      "tokens": 134759,
      "credits": 36,
      "model_cost_usd": 0.1833,
      "infra_cost_usd": 0.1036,
      "total_cost_usd": 0.287,
      "wall_seconds": 746.171394,
      "time_to_first_event_seconds": 6.137586,
      "tool_calls": 21,
      "approvals_granted": 0,
      "failure_reason": "patch_empty",
      "run_ids": [
        "743f3b520e3ae1ed9bdbdae0350af071",
        "f262bac88abf93f4c1b39cbde5e48261"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 22,
          "tokens": 134759,
          "credits": 22
        }
      ]
    },
    {
      "instance_id": "django__django-14787",
      "resolved": false,
      "scored": false,
      "tokens": 36557,
      "credits": 40,
      "model_cost_usd": 0.075,
      "infra_cost_usd": 0.2499,
      "total_cost_usd": 0.3249,
      "wall_seconds": 1798.999986012,
      "time_to_first_event_seconds": 10.545,
      "tool_calls": 10,
      "approvals_granted": 0,
      "failure_reason": "timeout",
      "run_ids": [
        "c8db1054741e1eda87bea19025b82e1c",
        "7a964d9b8a676c0d14d88969dabe36b9"
      ],
      "models": [
        {
          "model": "openai/gpt-5.6-luna",
          "calls": 7,
          "tokens": 29181,
          "credits": 7
        },
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 2,
          "tokens": 7376,
          "credits": 2
        }
      ]
    },
    {
      "instance_id": "django__django-16255",
      "resolved": false,
      "scored": false,
      "tokens": 224136,
      "credits": 61,
      "model_cost_usd": 0.2417,
      "infra_cost_usd": 0.25,
      "total_cost_usd": 0.4916,
      "wall_seconds": 1799.669707985,
      "time_to_first_event_seconds": 5.655713,
      "tool_calls": 28,
      "approvals_granted": 1,
      "failure_reason": "timeout",
      "run_ids": [
        "3ffe1adcb78cf9441362d946d462d9eb",
        "761ebe3c7dc61dc787203019b943f249",
        "29f9403038c5cdfc67979050451ce770"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 29,
          "tokens": 224136,
          "credits": 29
        }
      ]
    },
    {
      "instance_id": "django__django-16485",
      "resolved": true,
      "scored": true,
      "tokens": 670767,
      "credits": 61,
      "model_cost_usd": 0.3917,
      "infra_cost_usd": 0.0847,
      "total_cost_usd": 0.4764,
      "wall_seconds": 609.783425,
      "time_to_first_event_seconds": 5.551458,
      "tool_calls": 46,
      "approvals_granted": 0,
      "run_ids": [
        "1b71344c51bdebf9d4fcd274f9d71cb9"
      ],
      "models": [
        {
          "model": "openai/gpt-5.6-luna",
          "calls": 44,
          "tokens": 657279,
          "credits": 44
        },
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 3,
          "tokens": 13488,
          "credits": 3
        }
      ]
    },
    {
      "instance_id": "django__django-16661",
      "resolved": true,
      "scored": true,
      "tokens": 185091,
      "credits": 38,
      "model_cost_usd": 0.1917,
      "infra_cost_usd": 0.1022,
      "total_cost_usd": 0.2938,
      "wall_seconds": 735.493978,
      "time_to_first_event_seconds": 10.461452,
      "tool_calls": 22,
      "approvals_granted": 0,
      "run_ids": [
        "b1760d2dc1ba73605695df4d4a9d231e"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 23,
          "tokens": 185091,
          "credits": 23
        }
      ]
    },
    {
      "instance_id": "django__django-9296",
      "resolved": false,
      "scored": false,
      "tokens": 689663,
      "credits": 89,
      "model_cost_usd": 0.4833,
      "infra_cost_usd": 0.241,
      "total_cost_usd": 0.7244,
      "wall_seconds": 1735.441088,
      "time_to_first_event_seconds": 6.534545,
      "tool_calls": 58,
      "approvals_granted": 2,
      "failure_reason": "run_error",
      "run_ids": [
        "fabc725278b9935388f79b4f8ce9c02d",
        "5a6e5233e0f62df42b43cf63bdde630b",
        "4e817cf9d700b83b4d6d479621932e74",
        "03401fab766bdb487615f40274e47143"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 58,
          "tokens": 689663,
          "credits": 58
        }
      ]
    },
    {
      "instance_id": "matplotlib__matplotlib-20676",
      "resolved": false,
      "scored": false,
      "tokens": 1592237,
      "credits": 120,
      "model_cost_usd": 0.7417,
      "infra_cost_usd": 0.24,
      "total_cost_usd": 0.9817,
      "wall_seconds": 1727.99492,
      "time_to_first_event_seconds": 6.785015,
      "tool_calls": 88,
      "approvals_granted": 1,
      "failure_reason": "patch_empty",
      "run_ids": [
        "effc0b8b3c845d78d3d44a587e3b7c00",
        "002757f0d9b604d6a9d3ddd823ce21f2",
        "78c52e9a44aa39274890ccf7596fbdba"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 89,
          "tokens": 1592237,
          "credits": 89
        }
      ]
    },
    {
      "instance_id": "matplotlib__matplotlib-26466",
      "resolved": false,
      "scored": true,
      "tokens": 955773,
      "credits": 83,
      "model_cost_usd": 0.45,
      "infra_cost_usd": 0.25,
      "total_cost_usd": 0.7,
      "wall_seconds": 1799.702428271,
      "time_to_first_event_seconds": 5.21469,
      "tool_calls": 57,
      "approvals_granted": 1,
      "salvaged": true,
      "failure_reason": "timeout",
      "run_ids": [
        "3a64716d27b9ca1b1cbacd46e32c07e4",
        "c09081f6aea93d5f316a9b817c5f506d"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 48,
          "tokens": 924363,
          "credits": 48
        },
        {
          "model": "openai/gpt-5.6-luna",
          "calls": 6,
          "tokens": 31410,
          "credits": 6
        }
      ]
    },
    {
      "instance_id": "psf__requests-2931",
      "resolved": true,
      "scored": true,
      "tokens": 786059,
      "credits": 63,
      "model_cost_usd": 0.45,
      "infra_cost_usd": 0.0707,
      "total_cost_usd": 0.5207,
      "wall_seconds": 509.040706,
      "time_to_first_event_seconds": 4.551471,
      "tool_calls": 57,
      "approvals_granted": 0,
      "run_ids": [
        "faed64d9c29c699b27753ab4662055a6"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 54,
          "tokens": 786059,
          "credits": 54
        }
      ]
    },
    {
      "instance_id": "pydata__xarray-4695",
      "resolved": true,
      "scored": true,
      "tokens": 879817,
      "credits": 63,
      "model_cost_usd": 0.4083,
      "infra_cost_usd": 0.1016,
      "total_cost_usd": 0.51,
      "wall_seconds": 731.669355,
      "time_to_first_event_seconds": 5.291423,
      "tool_calls": 61,
      "approvals_granted": 0,
      "run_ids": [
        "64d78b18f9598f75fcd218857f308e4d"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 49,
          "tokens": 879817,
          "credits": 49
        }
      ]
    },
    {
      "instance_id": "pydata__xarray-6721",
      "resolved": false,
      "scored": true,
      "tokens": 1054441,
      "credits": 82,
      "model_cost_usd": 0.5917,
      "infra_cost_usd": 0.0789,
      "total_cost_usd": 0.6705,
      "wall_seconds": 567.729127,
      "time_to_first_event_seconds": 4.94402,
      "tool_calls": 68,
      "approvals_granted": 0,
      "run_ids": [
        "4d88610cb1ebf66d13bbbb64c4c80d68",
        "cb91397d73b05a0e702b9c3c8d0e62fa"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 71,
          "tokens": 1054441,
          "credits": 71
        }
      ]
    },
    {
      "instance_id": "scikit-learn__scikit-learn-25232",
      "resolved": true,
      "scored": true,
      "tokens": 1080897,
      "credits": 93,
      "model_cost_usd": 0.5167,
      "infra_cost_usd": 0.2298,
      "total_cost_usd": 0.7465,
      "wall_seconds": 1654.498781,
      "time_to_first_event_seconds": 4.632084,
      "tool_calls": 61,
      "approvals_granted": 1,
      "run_ids": [
        "d4d0002b49adef6fd5c71470397c57df",
        "27e07c1f514a0f8c30d244723955d13f",
        "c238cc433c23427a802660f132e18fab"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 62,
          "tokens": 1080897,
          "credits": 62
        }
      ]
    },
    {
      "instance_id": "scikit-learn__scikit-learn-25747",
      "resolved": true,
      "scored": true,
      "tokens": 1094949,
      "credits": 48,
      "model_cost_usd": 0.2667,
      "infra_cost_usd": 0.1077,
      "total_cost_usd": 0.3744,
      "wall_seconds": 775.56076,
      "time_to_first_event_seconds": 5.930676,
      "tool_calls": 50,
      "approvals_granted": 0,
      "run_ids": [
        "c5b90fea75232d3e431d176aa54008e8"
      ],
      "models": [
        {
          "model": "openai/gpt-5.6-luna",
          "calls": 28,
          "tokens": 1067363,
          "credits": 28
        },
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 4,
          "tokens": 27586,
          "credits": 4
        }
      ]
    },
    {
      "instance_id": "scikit-learn__scikit-learn-25931",
      "resolved": false,
      "scored": false,
      "tokens": 1059723,
      "credits": 89,
      "model_cost_usd": 0.55,
      "infra_cost_usd": 0.1653,
      "total_cost_usd": 0.7153,
      "wall_seconds": 1189.988564,
      "time_to_first_event_seconds": 5.513738,
      "tool_calls": 68,
      "approvals_granted": 0,
      "failure_reason": "patch_empty",
      "run_ids": [
        "4ddbab49edf5f227a6523f362bcda3b9",
        "9fa9753b1e67db2a4d94e3fcbbe80ff3"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 66,
          "tokens": 1059723,
          "credits": 66
        }
      ]
    },
    {
      "instance_id": "scikit-learn__scikit-learn-25973",
      "resolved": true,
      "scored": true,
      "tokens": 1210146,
      "credits": 85,
      "model_cost_usd": 0.5417,
      "infra_cost_usd": 0.1395,
      "total_cost_usd": 0.6812,
      "wall_seconds": 1004.688722,
      "time_to_first_event_seconds": 6.814463,
      "tool_calls": 65,
      "approvals_granted": 1,
      "run_ids": [
        "e8c6320b866c2ae7c39e202f08ac576f",
        "affa64b7c9386f0c52ff014d4181a79e"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 65,
          "tokens": 1210146,
          "credits": 65
        }
      ]
    },
    {
      "instance_id": "sphinx-doc__sphinx-10323",
      "resolved": true,
      "scored": true,
      "tokens": 1045758,
      "credits": 73,
      "model_cost_usd": 0.4333,
      "infra_cost_usd": 0.1643,
      "total_cost_usd": 0.5977,
      "wall_seconds": 1183.228757,
      "time_to_first_event_seconds": 4.516578,
      "tool_calls": 51,
      "approvals_granted": 0,
      "run_ids": [
        "c4d6c005ab96553588814b60f24f9bdc"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 52,
          "tokens": 1045758,
          "credits": 52
        }
      ]
    },
    {
      "instance_id": "sphinx-doc__sphinx-10673",
      "resolved": true,
      "scored": true,
      "tokens": 1081144,
      "credits": 52,
      "model_cost_usd": 0.3,
      "infra_cost_usd": 0.1123,
      "total_cost_usd": 0.4123,
      "wall_seconds": 808.292334,
      "time_to_first_event_seconds": 6.608669,
      "tool_calls": 46,
      "approvals_granted": 0,
      "run_ids": [
        "150bcf65762980a70ba7d7dfdd66a865"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 36,
          "tokens": 1081144,
          "credits": 36
        }
      ]
    },
    {
      "instance_id": "sphinx-doc__sphinx-11445",
      "resolved": false,
      "scored": false,
      "tokens": 766221,
      "credits": 84,
      "model_cost_usd": 0.425,
      "infra_cost_usd": 0.2499,
      "total_cost_usd": 0.6749,
      "wall_seconds": 1799.203025933,
      "time_to_first_event_seconds": 10.339501,
      "tool_calls": 54,
      "approvals_granted": 1,
      "failure_reason": "timeout",
      "run_ids": [
        "7c412b63659b8c2e66e530f50216c6a3",
        "c1da64fedc9ab81b00ffd4cb05fe7611",
        "1d00026ca5e5d69b039d60117921faab"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 51,
          "tokens": 766221,
          "credits": 51
        }
      ]
    },
    {
      "instance_id": "sphinx-doc__sphinx-11510",
      "resolved": false,
      "scored": false,
      "tokens": 1157995,
      "credits": 86,
      "model_cost_usd": 0.525,
      "infra_cost_usd": 0.1718,
      "total_cost_usd": 0.6968,
      "wall_seconds": 1237.115254,
      "time_to_first_event_seconds": 10.482459,
      "tool_calls": 62,
      "approvals_granted": 1,
      "failure_reason": "patch_empty",
      "run_ids": [
        "26f7683f802805e14152e03f8c54cd51",
        "cf108a1183d9d5da44988d56a43c8dc8",
        "68a49199781dbe5780a8a01c1d68d493"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 63,
          "tokens": 1157995,
          "credits": 63
        }
      ]
    },
    {
      "instance_id": "sphinx-doc__sphinx-9698",
      "resolved": true,
      "scored": true,
      "tokens": 278085,
      "credits": 37,
      "model_cost_usd": 0.1833,
      "infra_cost_usd": 0.0956,
      "total_cost_usd": 0.279,
      "wall_seconds": 688.583468,
      "time_to_first_event_seconds": 6.917641,
      "tool_calls": 27,
      "approvals_granted": 0,
      "run_ids": [
        "01df9e2455d6cdbdd550d5456ee44225"
      ],
      "models": [
        {
          "model": "openai/gpt-5.6-luna",
          "calls": 20,
          "tokens": 268922,
          "credits": 20
        },
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 2,
          "tokens": 9163,
          "credits": 2
        }
      ]
    },
    {
      "instance_id": "sympy__sympy-12096",
      "resolved": false,
      "scored": false,
      "tokens": 1044726,
      "credits": 93,
      "model_cost_usd": 0.5583,
      "infra_cost_usd": 0.2057,
      "total_cost_usd": 0.7641,
      "wall_seconds": 1481.174831,
      "time_to_first_event_seconds": 6.157032,
      "tool_calls": 67,
      "approvals_granted": 0,
      "failure_reason": "patch_empty",
      "run_ids": [
        "7319d5fe00fe12414c141b6d9f231cd0",
        "152c461f51883775d7018572ae270763"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 67,
          "tokens": 1044726,
          "credits": 67
        }
      ]
    },
    {
      "instance_id": "sympy__sympy-14531",
      "resolved": false,
      "scored": false,
      "tokens": 432290,
      "credits": 135,
      "model_cost_usd": 0.8667,
      "infra_cost_usd": 0.2499,
      "total_cost_usd": 1.1166,
      "wall_seconds": 1799.342080634,
      "time_to_first_event_seconds": 8.935561,
      "tool_calls": 47,
      "approvals_granted": 1,
      "failure_reason": "timeout",
      "run_ids": [
        "6359cf8885fdf69a9cf2b78ef366d636",
        "635374c824462919ca88afb73c6fbd68",
        "05b34455e8842de6c8010f65b252f0fc"
      ],
      "models": [
        {
          "model": "moonshotai/kimi-k3",
          "calls": 44,
          "tokens": 417879,
          "credits": 101
        },
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 3,
          "tokens": 14411,
          "credits": 3
        }
      ]
    },
    {
      "instance_id": "sympy__sympy-14711",
      "resolved": true,
      "scored": true,
      "tokens": 797201,
      "credits": 68,
      "model_cost_usd": 0.4083,
      "infra_cost_usd": 0.1261,
      "total_cost_usd": 0.5345,
      "wall_seconds": 908.174007,
      "time_to_first_event_seconds": 6.110265,
      "tool_calls": 50,
      "approvals_granted": 0,
      "run_ids": [
        "3d744facb212b709c9fc409dbc50409d"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 49,
          "tokens": 797201,
          "credits": 49
        }
      ]
    },
    {
      "instance_id": "sympy__sympy-17655",
      "resolved": true,
      "scored": true,
      "tokens": 1011968,
      "credits": 79,
      "model_cost_usd": 0.4667,
      "infra_cost_usd": 0.1588,
      "total_cost_usd": 0.6255,
      "wall_seconds": 1143.588289,
      "time_to_first_event_seconds": 6.383693,
      "tool_calls": 68,
      "approvals_granted": 3,
      "run_ids": [
        "3d6be402cbe62b560dafd98a152fe80d",
        "6e88d33c8ddd1d924a68cc600db532c9",
        "a79f1b5f6d814988d4c2357bb37e57e0",
        "c335bfe064aba9cef8f30aaa0d03e82c"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 56,
          "tokens": 1011968,
          "credits": 56
        }
      ]
    },
    {
      "instance_id": "sympy__sympy-19783",
      "resolved": true,
      "scored": true,
      "tokens": 1067385,
      "credits": 98,
      "model_cost_usd": 0.5667,
      "infra_cost_usd": 0.2256,
      "total_cost_usd": 0.7923,
      "wall_seconds": 1624.408575,
      "time_to_first_event_seconds": 5.465178,
      "tool_calls": 70,
      "approvals_granted": 0,
      "run_ids": [
        "2f26dc1c8aebf1f275eace7cf88fa5da",
        "3d55d485e2a9bc1e59aaecb0a5c72e43"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 68,
          "tokens": 1067385,
          "credits": 68
        }
      ]
    },
    {
      "instance_id": "sympy__sympy-21596",
      "resolved": false,
      "scored": false,
      "tokens": 1083424,
      "credits": 86,
      "model_cost_usd": 0.525,
      "infra_cost_usd": 0.1741,
      "total_cost_usd": 0.6991,
      "wall_seconds": 1253.364113,
      "time_to_first_event_seconds": 5.692226,
      "tool_calls": 62,
      "approvals_granted": 0,
      "failure_reason": "patch_empty",
      "run_ids": [
        "f2ea59858df2fb6f03dcef5304c88611",
        "eed52a00670ffb9ba3cfd59b7b5e8d74"
      ],
      "models": [
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 63,
          "tokens": 1083424,
          "credits": 63
        }
      ]
    },
    {
      "instance_id": "sympy__sympy-21930",
      "resolved": false,
      "scored": true,
      "tokens": 1044596,
      "credits": 129,
      "model_cost_usd": 0.9083,
      "infra_cost_usd": 0.163,
      "total_cost_usd": 1.0713,
      "wall_seconds": 1173.683632,
      "time_to_first_event_seconds": 5.211784,
      "tool_calls": 65,
      "approvals_granted": 0,
      "run_ids": [
        "b148c7bf7405ae090bbb920b36f0ec09"
      ],
      "models": [
        {
          "model": "moonshotai/kimi-k3",
          "calls": 14,
          "tokens": 359033,
          "credits": 60
        },
        {
          "model": "deepseek/deepseek-v4-flash-0731",
          "calls": 49,
          "tokens": 685563,
          "credits": 49
        }
      ]
    },
    {
      "instance_id": "sympy__sympy-23413",
      "resolved": true,
      "scored": true,
      "tokens": 1018609,
      "credits": 74,
      "model_cost_usd": 0.4333,
      "infra_cost_usd": 0.1601,
      "total_cost_usd": 0.5935,
      "wall_seconds": 1152.919823,
      "time_to_first_event_seconds": 9.91767,
      "tool_calls": 65,
      "approvals_granted": 0,
      "run_ids": [
        "8a6def5c691e5d1ace712c73fd9cb16b"
      ],
      "models": [
        {
          "model": "openai/gpt-5.6-luna",
          "calls": 52,
          "tokens": 1018609,
          "credits": 52
        }
      ]
    }
  ]
}