{ "experiment": "E7: SWE-bench Verified end-to-end task-success", "dataset": "princeton-nlp/SWE-bench_Verified", "split": "test", "seed": 1729, "n": 50, "agent": "aider 0.86.2", "model": "claude-sonnet-4-6", "temperature": 0.0, "harness": "swebench 4.1.0 (official run_evaluation, namespace=swebench, x86_64 under emulation)", "conditions": [ { "condition": "full", "n": 50, "resolved": 26, "pass_at_1": 0.52, "ci95_low": 0.3851, "ci95_high": 0.652, "cost_usd": 17.63, "total_input_tokens": 5557858, "total_output_tokens": 63840, "blocks_seen": 780, "blocks_compressed": 0, "context_char_reduction": 0.0, "per_instance": { "astropy__astropy-13236": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 50.0, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 147118, "usage_output_tokens": 1273, "cost_usd": 0.4604, "harness_completed": true, "harness_errored": false }, "django__django-10554": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 24.7, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 63915, "usage_output_tokens": 509, "cost_usd": 0.1994, "harness_completed": true, "harness_errored": false }, "django__django-10973": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 22.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 13913, "usage_output_tokens": 815, "cost_usd": 0.054, "harness_completed": true, "harness_errored": false }, "django__django-11066": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 38.5, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 173396, "usage_output_tokens": 685, "cost_usd": 0.5305, "harness_completed": true, "harness_errored": false }, "django__django-11087": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 93.8, "blocks_compressed": 0, "blocks_seen": 25, "usage_input_tokens": 672298, "usage_output_tokens": 2368, "cost_usd": 2.0524, "harness_completed": true, "harness_errored": false }, "django__django-11451": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 17.9, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 16477, "usage_output_tokens": 242, "cost_usd": 0.0531, "harness_completed": true, "harness_errored": false }, "django__django-12304": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 16.7, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 14295, "usage_output_tokens": 215, "cost_usd": 0.0461, "harness_completed": true, "harness_errored": false }, "django__django-12713": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 22.2, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 37881, "usage_output_tokens": 577, "cost_usd": 0.1223, "harness_completed": true, "harness_errored": false }, "django__django-13023": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 44.4, "blocks_compressed": 0, "blocks_seen": 34, "usage_input_tokens": 112507, "usage_output_tokens": 1365, "cost_usd": 0.358, "harness_completed": true, "harness_errored": false }, "django__django-13315": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 20.0, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 31458, "usage_output_tokens": 333, "cost_usd": 0.0994, "harness_completed": true, "harness_errored": false }, "django__django-13417": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.3, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 63645, "usage_output_tokens": 700, "cost_usd": 0.2014, "harness_completed": true, "harness_errored": false }, "django__django-14170": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 79.9, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 91291, "usage_output_tokens": 4337, "cost_usd": 0.3389, "harness_completed": true, "harness_errored": false }, "django__django-14315": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 50.4, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 87802, "usage_output_tokens": 1297, "cost_usd": 0.2829, "harness_completed": true, "harness_errored": false }, "django__django-15128": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 29.3, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 43394, "usage_output_tokens": 790, "cost_usd": 0.142, "harness_completed": true, "harness_errored": false }, "django__django-15161": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 29.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 28270, "usage_output_tokens": 1385, "cost_usd": 0.1056, "harness_completed": true, "harness_errored": false }, "django__django-15278": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 37.0, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 68848, "usage_output_tokens": 812, "cost_usd": 0.2187, "harness_completed": true, "harness_errored": false }, "django__django-15375": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 35.4, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 43530, "usage_output_tokens": 1037, "cost_usd": 0.1461, "harness_completed": true, "harness_errored": false }, "django__django-15695": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 37.0, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 147083, "usage_output_tokens": 1423, "cost_usd": 0.4626, "harness_completed": true, "harness_errored": false }, "django__django-15741": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 19.8, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 16535, "usage_output_tokens": 318, "cost_usd": 0.0544, "harness_completed": true, "harness_errored": false }, "django__django-15930": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 40.7, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 59628, "usage_output_tokens": 1544, "cost_usd": 0.202, "harness_completed": true, "harness_errored": false }, "django__django-16569": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.4, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 26342, "usage_output_tokens": 276, "cost_usd": 0.0832, "harness_completed": true, "harness_errored": false }, "django__django-16595": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 18.3, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 18178, "usage_output_tokens": 274, "cost_usd": 0.0586, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-20488": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 201.5, "blocks_compressed": 0, "blocks_seen": 25, "usage_input_tokens": 157548, "usage_output_tokens": 12448, "cost_usd": 0.6594, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-20676": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.7, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 47796, "usage_output_tokens": 609, "cost_usd": 0.1525, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-23476": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 41.6, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 51530, "usage_output_tokens": 2027, "cost_usd": 0.185, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-25775": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 38.2, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 78299, "usage_output_tokens": 1819, "cost_usd": 0.2622, "harness_completed": true, "harness_errored": false }, "psf__requests-2317": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 21.6, "blocks_compressed": 0, "blocks_seen": 24, "usage_input_tokens": 37247, "usage_output_tokens": 1492, "cost_usd": 0.1341, "harness_completed": true, "harness_errored": false }, "pydata__xarray-4966": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 55.5, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 120892, "usage_output_tokens": 2117, "cost_usd": 0.3944, "harness_completed": true, "harness_errored": false }, "pydata__xarray-6744": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 65.4, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 1589871, "usage_output_tokens": 1477, "cost_usd": 4.7918, "harness_completed": true, "harness_errored": false }, "pydata__xarray-7233": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 18.3, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 26840, "usage_output_tokens": 349, "cost_usd": 0.0858, "harness_completed": true, "harness_errored": false }, "pytest-dev__pytest-5631": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 18.1, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 26392, "usage_output_tokens": 566, "cost_usd": 0.0877, "harness_completed": true, "harness_errored": false }, "pytest-dev__pytest-7236": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 24.8, "blocks_compressed": 0, "blocks_seen": 21, "usage_input_tokens": 36493, "usage_output_tokens": 1086, "cost_usd": 0.1258, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-10908": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 12.9, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 28551, "usage_output_tokens": 226, "cost_usd": 0.089, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-13439": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 10.8, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 23179, "usage_output_tokens": 163, "cost_usd": 0.072, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-14496": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 29.4, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 73764, "usage_output_tokens": 985, "cost_usd": 0.2361, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-25232": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 23.3, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 23162, "usage_output_tokens": 911, "cost_usd": 0.0832, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-26194": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 49.7, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 394287, "usage_output_tokens": 1547, "cost_usd": 1.2061, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-7440": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 52.2, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 248146, "usage_output_tokens": 1351, "cost_usd": 0.7647, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-9591": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 22.0, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 44419, "usage_output_tokens": 596, "cost_usd": 0.1422, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-9711": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 39.1, "blocks_compressed": 0, "blocks_seen": 21, "usage_input_tokens": 78118, "usage_output_tokens": 1695, "cost_usd": 0.2598, "harness_completed": true, "harness_errored": false }, "sympy__sympy-12419": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 22.1, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 22455, "usage_output_tokens": 720, "cost_usd": 0.0782, "harness_completed": true, "harness_errored": false }, "sympy__sympy-13615": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 50.1, "blocks_compressed": 0, "blocks_seen": 33, "usage_input_tokens": 83561, "usage_output_tokens": 1596, "cost_usd": 0.2746, "harness_completed": true, "harness_errored": false }, "sympy__sympy-14248": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 36.1, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 88256, "usage_output_tokens": 1146, "cost_usd": 0.282, "harness_completed": true, "harness_errored": false }, "sympy__sympy-14531": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 26.0, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 23598, "usage_output_tokens": 747, "cost_usd": 0.082, "harness_completed": true, "harness_errored": false }, "sympy__sympy-15017": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 17.7, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 17697, "usage_output_tokens": 179, "cost_usd": 0.0558, "harness_completed": true, "harness_errored": false }, "sympy__sympy-15976": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 31.4, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 26024, "usage_output_tokens": 965, "cost_usd": 0.0925, "harness_completed": true, "harness_errored": false }, "sympy__sympy-16450": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 23.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 32362, "usage_output_tokens": 467, "cost_usd": 0.1041, "harness_completed": true, "harness_errored": false }, "sympy__sympy-17655": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 21.3, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 25356, "usage_output_tokens": 366, "cost_usd": 0.0816, "harness_completed": true, "harness_errored": false }, "sympy__sympy-20916": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 54.9, "blocks_compressed": 0, "blocks_seen": 31, "usage_input_tokens": 133791, "usage_output_tokens": 1960, "cost_usd": 0.4308, "harness_completed": true, "harness_errored": false }, "sympy__sympy-21596": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 42.0, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 40420, "usage_output_tokens": 1655, "cost_usd": 0.1461, "harness_completed": true, "harness_errored": false } } }, { "condition": "distil_trunc500", "n": 50, "resolved": 8, "pass_at_1": 0.16, "ci95_low": 0.0834, "ci95_high": 0.2851, "cost_usd": 4.0, "total_input_tokens": 1161769, "total_output_tokens": 34472, "blocks_seen": 1013, "blocks_compressed": 404, "context_char_reduction": 0.8552, "per_instance": { "astropy__astropy-13236": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 41.9, "blocks_compressed": 11, "blocks_seen": 23, "usage_input_tokens": 8131, "usage_output_tokens": 747, "cost_usd": 0.0356, "harness_completed": true, "harness_errored": false }, "django__django-10554": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 51.7, "blocks_compressed": 7, "blocks_seen": 22, "usage_input_tokens": 12782, "usage_output_tokens": 2080, "cost_usd": 0.0695, "harness_completed": true, "harness_errored": false }, "django__django-10973": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 28.8, "blocks_compressed": 11, "blocks_seen": 33, "usage_input_tokens": 11477, "usage_output_tokens": 811, "cost_usd": 0.0466, "harness_completed": true, "harness_errored": false }, "django__django-11066": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 35.1, "blocks_compressed": 5, "blocks_seen": 15, "usage_input_tokens": 158995, "usage_output_tokens": 467, "cost_usd": 0.484, "harness_completed": true, "harness_errored": false }, "django__django-11087": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 69.1, "blocks_compressed": 4, "blocks_seen": 15, "usage_input_tokens": 432935, "usage_output_tokens": 1178, "cost_usd": 1.3165, "harness_completed": true, "harness_errored": false }, "django__django-11451": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 23.8, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 6194, "usage_output_tokens": 218, "cost_usd": 0.0219, "harness_completed": true, "harness_errored": false }, "django__django-12304": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 16.3, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 5487, "usage_output_tokens": 233, "cost_usd": 0.02, "harness_completed": true, "harness_errored": false }, "django__django-12713": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 35.2, "blocks_compressed": 11, "blocks_seen": 33, "usage_input_tokens": 12369, "usage_output_tokens": 1356, "cost_usd": 0.0574, "harness_completed": true, "harness_errored": false }, "django__django-13023": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 28.7, "blocks_compressed": 7, "blocks_seen": 22, "usage_input_tokens": 34654, "usage_output_tokens": 592, "cost_usd": 0.1128, "harness_completed": true, "harness_errored": false }, "django__django-13315": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 21.6, "blocks_compressed": 7, "blocks_seen": 22, "usage_input_tokens": 7950, "usage_output_tokens": 371, "cost_usd": 0.0294, "harness_completed": true, "harness_errored": false }, "django__django-13417": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 26.2, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 5803, "usage_output_tokens": 768, "cost_usd": 0.0289, "harness_completed": true, "harness_errored": false }, "django__django-14170": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 57.0, "blocks_compressed": 7, "blocks_seen": 23, "usage_input_tokens": 53493, "usage_output_tokens": 2129, "cost_usd": 0.1924, "harness_completed": true, "harness_errored": false }, "django__django-14315": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 43.3, "blocks_compressed": 7, "blocks_seen": 23, "usage_input_tokens": 79593, "usage_output_tokens": 948, "cost_usd": 0.253, "harness_completed": true, "harness_errored": false }, "django__django-15128": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 39.6, "blocks_compressed": 11, "blocks_seen": 33, "usage_input_tokens": 15095, "usage_output_tokens": 1252, "cost_usd": 0.0641, "harness_completed": true, "harness_errored": false }, "django__django-15161": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 27.1, "blocks_compressed": 7, "blocks_seen": 21, "usage_input_tokens": 8153, "usage_output_tokens": 414, "cost_usd": 0.0307, "harness_completed": true, "harness_errored": false }, "django__django-15278": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 36.3, "blocks_compressed": 7, "blocks_seen": 23, "usage_input_tokens": 57465, "usage_output_tokens": 767, "cost_usd": 0.1839, "harness_completed": true, "harness_errored": false }, "django__django-15375": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 28.5, "blocks_compressed": 7, "blocks_seen": 22, "usage_input_tokens": 9685, "usage_output_tokens": 615, "cost_usd": 0.0383, "harness_completed": true, "harness_errored": false }, "django__django-15695": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 45.4, "blocks_compressed": 8, "blocks_seen": 22, "usage_input_tokens": 10844, "usage_output_tokens": 2011, "cost_usd": 0.0627, "harness_completed": true, "harness_errored": false }, "django__django-15741": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 18.1, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 5350, "usage_output_tokens": 218, "cost_usd": 0.0193, "harness_completed": true, "harness_errored": false }, "django__django-15930": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 27.6, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 5636, "usage_output_tokens": 843, "cost_usd": 0.0296, "harness_completed": true, "harness_errored": false }, "django__django-16569": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 26.4, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 12578, "usage_output_tokens": 179, "cost_usd": 0.0404, "harness_completed": true, "harness_errored": false }, "django__django-16595": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 31.9, "blocks_compressed": 11, "blocks_seen": 33, "usage_input_tokens": 12813, "usage_output_tokens": 856, "cost_usd": 0.0513, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-20488": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 28.1, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 7996, "usage_output_tokens": 761, "cost_usd": 0.0354, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-20676": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 30.4, "blocks_compressed": 14, "blocks_seen": 33, "usage_input_tokens": 10764, "usage_output_tokens": 762, "cost_usd": 0.0437, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-23476": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 32.4, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 8041, "usage_output_tokens": 1057, "cost_usd": 0.04, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-25775": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 26.7, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 7842, "usage_output_tokens": 700, "cost_usd": 0.034, "harness_completed": true, "harness_errored": false }, "psf__requests-2317": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 7.0, "blocks_compressed": 5, "blocks_seen": 14, "usage_input_tokens": 5762, "usage_output_tokens": 148, "cost_usd": 0.0195, "harness_completed": true, "harness_errored": false }, "pydata__xarray-4966": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 18.1, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 4844, "usage_output_tokens": 306, "cost_usd": 0.0191, "harness_completed": true, "harness_errored": false }, "pydata__xarray-6744": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.9, "blocks_compressed": 11, "blocks_seen": 23, "usage_input_tokens": 8071, "usage_output_tokens": 711, "cost_usd": 0.0349, "harness_completed": true, "harness_errored": false }, "pydata__xarray-7233": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 15.1, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 4683, "usage_output_tokens": 177, "cost_usd": 0.0167, "harness_completed": true, "harness_errored": false }, "pytest-dev__pytest-5631": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 12.3, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 4706, "usage_output_tokens": 233, "cost_usd": 0.0176, "harness_completed": true, "harness_errored": false }, "pytest-dev__pytest-7236": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 17.0, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 7517, "usage_output_tokens": 321, "cost_usd": 0.0274, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-10908": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 19.3, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 7604, "usage_output_tokens": 570, "cost_usd": 0.0314, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-13439": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 10.4, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 4653, "usage_output_tokens": 175, "cost_usd": 0.0166, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-14496": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 22.0, "blocks_compressed": 8, "blocks_seen": 15, "usage_input_tokens": 5433, "usage_output_tokens": 468, "cost_usd": 0.0233, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-25232": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 31.7, "blocks_compressed": 15, "blocks_seen": 33, "usage_input_tokens": 11870, "usage_output_tokens": 1416, "cost_usd": 0.0569, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-26194": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 29.8, "blocks_compressed": 8, "blocks_seen": 15, "usage_input_tokens": 5444, "usage_output_tokens": 780, "cost_usd": 0.028, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-7440": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 46.9, "blocks_compressed": 19, "blocks_seen": 37, "usage_input_tokens": 12128, "usage_output_tokens": 1063, "cost_usd": 0.0523, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-9591": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.6, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 7937, "usage_output_tokens": 937, "cost_usd": 0.0379, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-9711": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 15.7, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 4626, "usage_output_tokens": 248, "cost_usd": 0.0176, "harness_completed": true, "harness_errored": false }, "sympy__sympy-12419": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 8.9, "blocks_compressed": 2, "blocks_seen": 6, "usage_input_tokens": 2215, "usage_output_tokens": 30, "cost_usd": 0.0071, "harness_completed": true, "harness_errored": false }, "sympy__sympy-13615": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.3, "blocks_compressed": 8, "blocks_seen": 22, "usage_input_tokens": 7528, "usage_output_tokens": 562, "cost_usd": 0.031, "harness_completed": true, "harness_errored": false }, "sympy__sympy-14248": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 32.4, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 7854, "usage_output_tokens": 500, "cost_usd": 0.0311, "harness_completed": true, "harness_errored": false }, "sympy__sympy-14531": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 17.9, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 4820, "usage_output_tokens": 212, "cost_usd": 0.0176, "harness_completed": true, "harness_errored": false }, "sympy__sympy-15017": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.7, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 7639, "usage_output_tokens": 457, "cost_usd": 0.0298, "harness_completed": true, "harness_errored": false }, "sympy__sympy-15976": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 22.8, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 4761, "usage_output_tokens": 331, "cost_usd": 0.0192, "harness_completed": true, "harness_errored": false }, "sympy__sympy-16450": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 24.5, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 7380, "usage_output_tokens": 380, "cost_usd": 0.0278, "harness_completed": true, "harness_errored": false }, "sympy__sympy-17655": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 45.5, "blocks_compressed": 19, "blocks_seen": 37, "usage_input_tokens": 12587, "usage_output_tokens": 1464, "cost_usd": 0.0597, "harness_completed": true, "harness_errored": false }, "sympy__sympy-20916": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 23.4, "blocks_compressed": 5, "blocks_seen": 13, "usage_input_tokens": 4641, "usage_output_tokens": 322, "cost_usd": 0.0188, "harness_completed": true, "harness_errored": false }, "sympy__sympy-21596": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 21.6, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 4941, "usage_output_tokens": 328, "cost_usd": 0.0197, "harness_completed": true, "harness_errored": false } } }, { "condition": "llmlingua2", "n": 50, "resolved": 13, "pass_at_1": 0.26, "ci95_low": 0.1587, "ci95_high": 0.3955, "cost_usd": 12.03, "total_input_tokens": 3735294, "total_output_tokens": 54754, "blocks_seen": 1039, "blocks_compressed": 415, "context_char_reduction": 0.4831, "per_instance": { "astropy__astropy-13236": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 91.6, "blocks_compressed": 8, "blocks_seen": 15, "usage_input_tokens": 48285, "usage_output_tokens": 786, "cost_usd": 0.1566, "harness_completed": true, "harness_errored": false }, "django__django-10554": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 90.6, "blocks_compressed": 7, "blocks_seen": 22, "usage_input_tokens": 61716, "usage_output_tokens": 994, "cost_usd": 0.2001, "harness_completed": true, "harness_errored": false }, "django__django-10973": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 47.3, "blocks_compressed": 7, "blocks_seen": 22, "usage_input_tokens": 15001, "usage_output_tokens": 1170, "cost_usd": 0.0626, "harness_completed": true, "harness_errored": false }, "django__django-11066": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 49.7, "blocks_compressed": 5, "blocks_seen": 15, "usage_input_tokens": 166080, "usage_output_tokens": 589, "cost_usd": 0.5071, "harness_completed": true, "harness_errored": false }, "django__django-11087": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 63.7, "blocks_compressed": 4, "blocks_seen": 15, "usage_input_tokens": 438460, "usage_output_tokens": 1229, "cost_usd": 1.3338, "harness_completed": true, "harness_errored": false }, "django__django-11451": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 20.5, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 10918, "usage_output_tokens": 222, "cost_usd": 0.0361, "harness_completed": true, "harness_errored": false }, "django__django-12304": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 21.4, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 9486, "usage_output_tokens": 243, "cost_usd": 0.0321, "harness_completed": true, "harness_errored": false }, "django__django-12713": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 40.9, "blocks_compressed": 7, "blocks_seen": 22, "usage_input_tokens": 36822, "usage_output_tokens": 813, "cost_usd": 0.1227, "harness_completed": true, "harness_errored": false }, "django__django-13023": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 52.1, "blocks_compressed": 7, "blocks_seen": 23, "usage_input_tokens": 71787, "usage_output_tokens": 1105, "cost_usd": 0.2319, "harness_completed": true, "harness_errored": false }, "django__django-13315": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 46.0, "blocks_compressed": 7, "blocks_seen": 22, "usage_input_tokens": 30618, "usage_output_tokens": 632, "cost_usd": 0.1013, "harness_completed": true, "harness_errored": false }, "django__django-13417": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 51.7, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 32978, "usage_output_tokens": 624, "cost_usd": 0.1083, "harness_completed": true, "harness_errored": false }, "django__django-14170": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 75.3, "blocks_compressed": 4, "blocks_seen": 15, "usage_input_tokens": 68433, "usage_output_tokens": 2291, "cost_usd": 0.2397, "harness_completed": true, "harness_errored": false }, "django__django-14315": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 57.2, "blocks_compressed": 7, "blocks_seen": 23, "usage_input_tokens": 87716, "usage_output_tokens": 1500, "cost_usd": 0.2856, "harness_completed": true, "harness_errored": false }, "django__django-15128": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 39.7, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 24210, "usage_output_tokens": 659, "cost_usd": 0.0825, "harness_completed": true, "harness_errored": false }, "django__django-15161": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 65.2, "blocks_compressed": 11, "blocks_seen": 33, "usage_input_tokens": 40586, "usage_output_tokens": 2390, "cost_usd": 0.1576, "harness_completed": true, "harness_errored": false }, "django__django-15278": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 76.7, "blocks_compressed": 11, "blocks_seen": 37, "usage_input_tokens": 135663, "usage_output_tokens": 2271, "cost_usd": 0.4411, "harness_completed": true, "harness_errored": false }, "django__django-15375": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 52.7, "blocks_compressed": 7, "blocks_seen": 22, "usage_input_tokens": 42499, "usage_output_tokens": 1148, "cost_usd": 0.1447, "harness_completed": true, "harness_errored": false }, "django__django-15695": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 161.6, "blocks_compressed": 12, "blocks_seen": 33, "usage_input_tokens": 158154, "usage_output_tokens": 3045, "cost_usd": 0.5201, "harness_completed": true, "harness_errored": false }, "django__django-15741": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 33.2, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 13874, "usage_output_tokens": 246, "cost_usd": 0.0453, "harness_completed": true, "harness_errored": false }, "django__django-15930": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 61.5, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 30123, "usage_output_tokens": 1159, "cost_usd": 0.1078, "harness_completed": true, "harness_errored": false }, "django__django-16569": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 34.4, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 18855, "usage_output_tokens": 193, "cost_usd": 0.0595, "harness_completed": true, "harness_errored": false }, "django__django-16595": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 26.7, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 11548, "usage_output_tokens": 467, "cost_usd": 0.0416, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-20488": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 87.3, "blocks_compressed": 15, "blocks_seen": 33, "usage_input_tokens": 55716, "usage_output_tokens": 1206, "cost_usd": 0.1852, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-20676": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 86.4, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 47788, "usage_output_tokens": 2846, "cost_usd": 0.1861, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-23476": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 77.3, "blocks_compressed": 15, "blocks_seen": 33, "usage_input_tokens": 70674, "usage_output_tokens": 1165, "cost_usd": 0.2295, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-25775": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 97.8, "blocks_compressed": 15, "blocks_seen": 33, "usage_input_tokens": 79859, "usage_output_tokens": 1536, "cost_usd": 0.2626, "harness_completed": true, "harness_errored": false }, "psf__requests-2317": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 20.5, "blocks_compressed": 5, "blocks_seen": 14, "usage_input_tokens": 13490, "usage_output_tokens": 186, "cost_usd": 0.0433, "harness_completed": false, "harness_errored": true }, "pydata__xarray-4966": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 172.4, "blocks_compressed": 11, "blocks_seen": 23, "usage_input_tokens": 63406, "usage_output_tokens": 1676, "cost_usd": 0.2154, "harness_completed": true, "harness_errored": false }, "pydata__xarray-6744": { "resolved": false, "empty_patch": false, "agent_status": "timeout", "agent_seconds": 900.0, "blocks_compressed": 11, "blocks_seen": 24, "usage_input_tokens": 804018, "usage_output_tokens": 1594, "cost_usd": 2.436, "harness_completed": true, "harness_errored": false }, "pydata__xarray-7233": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 50.9, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 23776, "usage_output_tokens": 454, "cost_usd": 0.0781, "harness_completed": true, "harness_errored": false }, "pytest-dev__pytest-5631": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 40.0, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 12856, "usage_output_tokens": 254, "cost_usd": 0.0424, "harness_completed": true, "harness_errored": false }, "pytest-dev__pytest-7236": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 27.9, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 9823, "usage_output_tokens": 316, "cost_usd": 0.0342, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-10908": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 43.2, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 15401, "usage_output_tokens": 392, "cost_usd": 0.0521, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-13439": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 27.8, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 12324, "usage_output_tokens": 202, "cost_usd": 0.04, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-14496": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 65.7, "blocks_compressed": 8, "blocks_seen": 15, "usage_input_tokens": 33821, "usage_output_tokens": 682, "cost_usd": 0.1117, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-25232": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 50.6, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 23933, "usage_output_tokens": 1415, "cost_usd": 0.093, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-26194": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 647.4, "blocks_compressed": 13, "blocks_seen": 25, "usage_input_tokens": 303111, "usage_output_tokens": 2220, "cost_usd": 0.9426, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-7440": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 463.3, "blocks_compressed": 19, "blocks_seen": 37, "usage_input_tokens": 195948, "usage_output_tokens": 2216, "cost_usd": 0.6211, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-9591": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 55.5, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 23104, "usage_output_tokens": 399, "cost_usd": 0.0753, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-9711": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 51.1, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 20821, "usage_output_tokens": 522, "cost_usd": 0.0703, "harness_completed": true, "harness_errored": false }, "sympy__sympy-12419": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 104.5, "blocks_compressed": 15, "blocks_seen": 33, "usage_input_tokens": 48757, "usage_output_tokens": 2682, "cost_usd": 0.1865, "harness_completed": true, "harness_errored": false }, "sympy__sympy-13615": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 108.9, "blocks_compressed": 12, "blocks_seen": 33, "usage_input_tokens": 42153, "usage_output_tokens": 1212, "cost_usd": 0.1446, "harness_completed": true, "harness_errored": false }, "sympy__sympy-14248": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 127.6, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 79352, "usage_output_tokens": 1751, "cost_usd": 0.2643, "harness_completed": true, "harness_errored": false }, "sympy__sympy-14531": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 50.2, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 12792, "usage_output_tokens": 564, "cost_usd": 0.0468, "harness_completed": true, "harness_errored": false }, "sympy__sympy-15017": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 35.4, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 9976, "usage_output_tokens": 181, "cost_usd": 0.0326, "harness_completed": true, "harness_errored": false }, "sympy__sympy-15976": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 38.7, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 13291, "usage_output_tokens": 1216, "cost_usd": 0.0581, "harness_completed": true, "harness_errored": false }, "sympy__sympy-16450": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 42.8, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 30548, "usage_output_tokens": 631, "cost_usd": 0.1011, "harness_completed": true, "harness_errored": false }, "sympy__sympy-17655": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 55.3, "blocks_compressed": 15, "blocks_seen": 33, "usage_input_tokens": 32148, "usage_output_tokens": 987, "cost_usd": 0.1112, "harness_completed": true, "harness_errored": false }, "sympy__sympy-20916": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 80.8, "blocks_compressed": 11, "blocks_seen": 31, "usage_input_tokens": 65054, "usage_output_tokens": 1419, "cost_usd": 0.2164, "harness_completed": true, "harness_errored": false }, "sympy__sympy-21596": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 61.7, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 37542, "usage_output_tokens": 1054, "cost_usd": 0.1284, "harness_completed": true, "harness_errored": false } } }, { "condition": "distil_expand", "n": 50, "resolved": 28, "pass_at_1": 0.56, "ci95_low": 0.4231, "ci95_high": 0.6884, "cost_usd": 16.38, "total_input_tokens": 5085065, "total_output_tokens": 74721, "blocks_seen": 795, "blocks_compressed": 316, "context_char_reduction": 0.8103, "per_instance": { "astropy__astropy-13236": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 60.6, "blocks_compressed": 8, "blocks_seen": 15, "usage_input_tokens": 111793, "usage_output_tokens": 1408, "cost_usd": 0.3565, "harness_completed": true, "harness_errored": false }, "django__django-10554": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 29.8, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 39668, "usage_output_tokens": 700, "cost_usd": 0.1295, "harness_completed": true, "harness_errored": false }, "django__django-10973": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 24.5, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 10474, "usage_output_tokens": 868, "cost_usd": 0.0444, "harness_completed": true, "harness_errored": false }, "django__django-11066": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 50.9, "blocks_compressed": 5, "blocks_seen": 15, "usage_input_tokens": 254922, "usage_output_tokens": 928, "cost_usd": 0.7787, "harness_completed": true, "harness_errored": false }, "django__django-11087": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 81.0, "blocks_compressed": 4, "blocks_seen": 15, "usage_input_tokens": 549825, "usage_output_tokens": 1792, "cost_usd": 1.6764, "harness_completed": true, "harness_errored": false }, "django__django-11451": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 28.8, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 13261, "usage_output_tokens": 375, "cost_usd": 0.0454, "harness_completed": true, "harness_errored": false }, "django__django-12304": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 19.7, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 10828, "usage_output_tokens": 331, "cost_usd": 0.0374, "harness_completed": true, "harness_errored": false }, "django__django-12713": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.5, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 34432, "usage_output_tokens": 739, "cost_usd": 0.1144, "harness_completed": true, "harness_errored": false }, "django__django-13023": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 78.8, "blocks_compressed": 11, "blocks_seen": 37, "usage_input_tokens": 185185, "usage_output_tokens": 3341, "cost_usd": 0.6057, "harness_completed": true, "harness_errored": false }, "django__django-13315": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 26.3, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 28518, "usage_output_tokens": 715, "cost_usd": 0.0963, "harness_completed": true, "harness_errored": false }, "django__django-13417": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 28.1, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 32360, "usage_output_tokens": 689, "cost_usd": 0.1074, "harness_completed": true, "harness_errored": false }, "django__django-14170": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 118.0, "blocks_compressed": 7, "blocks_seen": 23, "usage_input_tokens": 169310, "usage_output_tokens": 6326, "cost_usd": 0.6028, "harness_completed": true, "harness_errored": false }, "django__django-14315": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 49.9, "blocks_compressed": 4, "blocks_seen": 15, "usage_input_tokens": 104418, "usage_output_tokens": 1408, "cost_usd": 0.3344, "harness_completed": true, "harness_errored": false }, "django__django-15128": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 29.4, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 40571, "usage_output_tokens": 744, "cost_usd": 0.1329, "harness_completed": true, "harness_errored": false }, "django__django-15161": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 46.3, "blocks_compressed": 7, "blocks_seen": 21, "usage_input_tokens": 69022, "usage_output_tokens": 1921, "cost_usd": 0.2359, "harness_completed": true, "harness_errored": false }, "django__django-15278": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 40.9, "blocks_compressed": 4, "blocks_seen": 15, "usage_input_tokens": 78929, "usage_output_tokens": 1023, "cost_usd": 0.2521, "harness_completed": true, "harness_errored": false }, "django__django-15375": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 70.5, "blocks_compressed": 7, "blocks_seen": 21, "usage_input_tokens": 115838, "usage_output_tokens": 2433, "cost_usd": 0.384, "harness_completed": true, "harness_errored": false }, "django__django-15695": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 55.7, "blocks_compressed": 5, "blocks_seen": 13, "usage_input_tokens": 151748, "usage_output_tokens": 2416, "cost_usd": 0.4915, "harness_completed": true, "harness_errored": false }, "django__django-15741": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 24.3, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 13016, "usage_output_tokens": 587, "cost_usd": 0.0479, "harness_completed": true, "harness_errored": false }, "django__django-15930": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 66.1, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 27300, "usage_output_tokens": 3600, "cost_usd": 0.1359, "harness_completed": true, "harness_errored": false }, "django__django-16569": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 22.6, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 26341, "usage_output_tokens": 253, "cost_usd": 0.0828, "harness_completed": true, "harness_errored": false }, "django__django-16595": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 22.8, "blocks_compressed": 4, "blocks_seen": 13, "usage_input_tokens": 14590, "usage_output_tokens": 464, "cost_usd": 0.0507, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-20488": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 40.4, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 57239, "usage_output_tokens": 915, "cost_usd": 0.1854, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-20676": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 47.1, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 120098, "usage_output_tokens": 1365, "cost_usd": 0.3808, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-23476": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 32.2, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 16278, "usage_output_tokens": 1000, "cost_usd": 0.0638, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-25775": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 31.8, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 38904, "usage_output_tokens": 955, "cost_usd": 0.131, "harness_completed": true, "harness_errored": false }, "psf__requests-2317": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 29.2, "blocks_compressed": 5, "blocks_seen": 14, "usage_input_tokens": 28193, "usage_output_tokens": 2129, "cost_usd": 0.1165, "harness_completed": false, "harness_errored": true }, "pydata__xarray-4966": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 67.2, "blocks_compressed": 8, "blocks_seen": 15, "usage_input_tokens": 124251, "usage_output_tokens": 2661, "cost_usd": 0.4127, "harness_completed": true, "harness_errored": false }, "pydata__xarray-6744": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 70.0, "blocks_compressed": 8, "blocks_seen": 15, "usage_input_tokens": 1202438, "usage_output_tokens": 1670, "cost_usd": 3.6324, "harness_completed": true, "harness_errored": false }, "pydata__xarray-7233": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 27.9, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 37210, "usage_output_tokens": 995, "cost_usd": 0.1266, "harness_completed": true, "harness_errored": false }, "pytest-dev__pytest-5631": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 21.1, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 19758, "usage_output_tokens": 519, "cost_usd": 0.0671, "harness_completed": true, "harness_errored": false }, "pytest-dev__pytest-7236": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 20.2, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 17899, "usage_output_tokens": 698, "cost_usd": 0.0642, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-10908": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 18.7, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 24107, "usage_output_tokens": 579, "cost_usd": 0.081, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-13439": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 14.1, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 18085, "usage_output_tokens": 246, "cost_usd": 0.0579, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-14496": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 38.0, "blocks_compressed": 8, "blocks_seen": 15, "usage_input_tokens": 49890, "usage_output_tokens": 1230, "cost_usd": 0.1681, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-25232": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 32.6, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 37051, "usage_output_tokens": 1123, "cost_usd": 0.128, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-26194": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 65.1, "blocks_compressed": 11, "blocks_seen": 23, "usage_input_tokens": 382330, "usage_output_tokens": 2416, "cost_usd": 1.1832, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-7440": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 63.9, "blocks_compressed": 8, "blocks_seen": 15, "usage_input_tokens": 138679, "usage_output_tokens": 2028, "cost_usd": 0.4465, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-9591": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 27.9, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 44070, "usage_output_tokens": 828, "cost_usd": 0.1446, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-9711": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 20.6, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 22156, "usage_output_tokens": 495, "cost_usd": 0.0739, "harness_completed": true, "harness_errored": false }, "sympy__sympy-12419": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 105.9, "blocks_compressed": 14, "blocks_seen": 30, "usage_input_tokens": 140907, "usage_output_tokens": 5232, "cost_usd": 0.5012, "harness_completed": true, "harness_errored": false }, "sympy__sympy-13615": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 77.2, "blocks_compressed": 12, "blocks_seen": 33, "usage_input_tokens": 89293, "usage_output_tokens": 3358, "cost_usd": 0.3182, "harness_completed": true, "harness_errored": false }, "sympy__sympy-14248": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 54.5, "blocks_compressed": 10, "blocks_seen": 22, "usage_input_tokens": 160598, "usage_output_tokens": 1534, "cost_usd": 0.5048, "harness_completed": true, "harness_errored": false }, "sympy__sympy-14531": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 33.5, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 24582, "usage_output_tokens": 1269, "cost_usd": 0.0928, "harness_completed": true, "harness_errored": false }, "sympy__sympy-15017": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 57.8, "blocks_compressed": 15, "blocks_seen": 33, "usage_input_tokens": 60018, "usage_output_tokens": 1972, "cost_usd": 0.2096, "harness_completed": true, "harness_errored": false }, "sympy__sympy-15976": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 38.5, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 19495, "usage_output_tokens": 1650, "cost_usd": 0.0832, "harness_completed": true, "harness_errored": false }, "sympy__sympy-16450": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 41.7, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 33185, "usage_output_tokens": 716, "cost_usd": 0.1103, "harness_completed": true, "harness_errored": false }, "sympy__sympy-17655": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 31.2, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 39392, "usage_output_tokens": 782, "cost_usd": 0.1299, "harness_completed": true, "harness_errored": false }, "sympy__sympy-20916": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 63.5, "blocks_compressed": 5, "blocks_seen": 13, "usage_input_tokens": 27908, "usage_output_tokens": 2179, "cost_usd": 0.1164, "harness_completed": true, "harness_errored": false }, "sympy__sympy-21596": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 38.5, "blocks_compressed": 6, "blocks_seen": 13, "usage_input_tokens": 28702, "usage_output_tokens": 1116, "cost_usd": 0.1028, "harness_completed": true, "harness_errored": false } } }, { "condition": "distil_gated", "n": 50, "resolved": 27, "pass_at_1": 0.54, "ci95_low": 0.404, "ci95_high": 0.6703, "cost_usd": 17.27, "total_input_tokens": 5442980, "total_output_tokens": 62616, "blocks_seen": 760, "blocks_compressed": 1, "context_char_reduction": 0.0004, "per_instance": { "astropy__astropy-13236": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 80.9, "blocks_compressed": 0, "blocks_seen": 23, "usage_input_tokens": 175708, "usage_output_tokens": 3162, "cost_usd": 0.5746, "harness_completed": true, "harness_errored": false }, "django__django-10554": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.6, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 55448, "usage_output_tokens": 542, "cost_usd": 0.1745, "harness_completed": true, "harness_errored": false }, "django__django-10973": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 28.8, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 16156, "usage_output_tokens": 1114, "cost_usd": 0.0652, "harness_completed": true, "harness_errored": false }, "django__django-11066": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 35.7, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 175887, "usage_output_tokens": 607, "cost_usd": 0.5368, "harness_completed": true, "harness_errored": false }, "django__django-11087": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 67.9, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 448307, "usage_output_tokens": 1493, "cost_usd": 1.3673, "harness_completed": true, "harness_errored": false }, "django__django-11451": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 17.9, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 17987, "usage_output_tokens": 246, "cost_usd": 0.0577, "harness_completed": true, "harness_errored": false }, "django__django-12304": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 18.0, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 15432, "usage_output_tokens": 301, "cost_usd": 0.0508, "harness_completed": true, "harness_errored": false }, "django__django-12713": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 22.2, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 39333, "usage_output_tokens": 642, "cost_usd": 0.1276, "harness_completed": true, "harness_errored": false }, "django__django-13023": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 44.4, "blocks_compressed": 0, "blocks_seen": 23, "usage_input_tokens": 86238, "usage_output_tokens": 1182, "cost_usd": 0.2764, "harness_completed": true, "harness_errored": false }, "django__django-13315": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 20.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 32811, "usage_output_tokens": 344, "cost_usd": 0.1036, "harness_completed": true, "harness_errored": false }, "django__django-13417": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 21.7, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 36642, "usage_output_tokens": 434, "cost_usd": 0.1164, "harness_completed": true, "harness_errored": false }, "django__django-14170": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 170.2, "blocks_compressed": 0, "blocks_seen": 23, "usage_input_tokens": 164171, "usage_output_tokens": 12602, "cost_usd": 0.6815, "harness_completed": true, "harness_errored": false }, "django__django-14315": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 49.0, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 90809, "usage_output_tokens": 1484, "cost_usd": 0.2947, "harness_completed": true, "harness_errored": false }, "django__django-15128": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 38.4, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 44708, "usage_output_tokens": 1550, "cost_usd": 0.1574, "harness_completed": true, "harness_errored": false }, "django__django-15161": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 28.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 29386, "usage_output_tokens": 1177, "cost_usd": 0.1058, "harness_completed": true, "harness_errored": false }, "django__django-15278": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 39.9, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 73734, "usage_output_tokens": 1060, "cost_usd": 0.2371, "harness_completed": true, "harness_errored": false }, "django__django-15375": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 31.0, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 44426, "usage_output_tokens": 794, "cost_usd": 0.1452, "harness_completed": true, "harness_errored": false }, "django__django-15695": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 36.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 148180, "usage_output_tokens": 1470, "cost_usd": 0.4666, "harness_completed": true, "harness_errored": false }, "django__django-15741": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 21.7, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 18387, "usage_output_tokens": 409, "cost_usd": 0.0613, "harness_completed": true, "harness_errored": false }, "django__django-15930": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 33.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 61676, "usage_output_tokens": 910, "cost_usd": 0.1987, "harness_completed": true, "harness_errored": false }, "django__django-16569": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 22.4, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 28156, "usage_output_tokens": 299, "cost_usd": 0.089, "harness_completed": true, "harness_errored": false }, "django__django-16595": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 19.2, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 19382, "usage_output_tokens": 317, "cost_usd": 0.0629, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-20488": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.7, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 57974, "usage_output_tokens": 611, "cost_usd": 0.1831, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-20676": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 26.6, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 49090, "usage_output_tokens": 647, "cost_usd": 0.157, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-23476": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 23.1, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 53419, "usage_output_tokens": 445, "cost_usd": 0.1669, "harness_completed": true, "harness_errored": false }, "matplotlib__matplotlib-25775": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 55.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 79039, "usage_output_tokens": 3346, "cost_usd": 0.2873, "harness_completed": true, "harness_errored": false }, "psf__requests-2317": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 21.8, "blocks_compressed": 1, "blocks_seen": 24, "usage_input_tokens": 36907, "usage_output_tokens": 1501, "cost_usd": 0.1332, "harness_completed": false, "harness_errored": true }, "pydata__xarray-4966": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 52.8, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 122808, "usage_output_tokens": 2284, "cost_usd": 0.4027, "harness_completed": true, "harness_errored": false }, "pydata__xarray-6744": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 62.0, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 1591145, "usage_output_tokens": 1203, "cost_usd": 4.7915, "harness_completed": true, "harness_errored": false }, "pydata__xarray-7233": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 30.5, "blocks_compressed": 0, "blocks_seen": 22, "usage_input_tokens": 74889, "usage_output_tokens": 750, "cost_usd": 0.2359, "harness_completed": true, "harness_errored": false }, "pytest-dev__pytest-5631": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 12.0, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 24203, "usage_output_tokens": 281, "cost_usd": 0.0768, "harness_completed": true, "harness_errored": false }, "pytest-dev__pytest-7236": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 25.6, "blocks_compressed": 0, "blocks_seen": 21, "usage_input_tokens": 38657, "usage_output_tokens": 1167, "cost_usd": 0.1335, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-10908": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 13.6, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 29363, "usage_output_tokens": 257, "cost_usd": 0.0919, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-13439": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 12.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 24539, "usage_output_tokens": 198, "cost_usd": 0.0766, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-14496": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 28.9, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 76626, "usage_output_tokens": 1003, "cost_usd": 0.2449, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-25232": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 26.3, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 25409, "usage_output_tokens": 1111, "cost_usd": 0.0929, "harness_completed": true, "harness_errored": false }, "scikit-learn__scikit-learn-26194": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 50.6, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 396271, "usage_output_tokens": 1645, "cost_usd": 1.2135, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-7440": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 53.3, "blocks_compressed": 0, "blocks_seen": 15, "usage_input_tokens": 251881, "usage_output_tokens": 1640, "cost_usd": 0.7802, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-9591": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 19.2, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 46457, "usage_output_tokens": 488, "cost_usd": 0.1467, "harness_completed": true, "harness_errored": false }, "sphinx-doc__sphinx-9711": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 54.2, "blocks_compressed": 0, "blocks_seen": 30, "usage_input_tokens": 162719, "usage_output_tokens": 2127, "cost_usd": 0.5201, "harness_completed": true, "harness_errored": false }, "sympy__sympy-12419": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 20.2, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 23566, "usage_output_tokens": 587, "cost_usd": 0.0795, "harness_completed": true, "harness_errored": false }, "sympy__sympy-13615": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 40.5, "blocks_compressed": 0, "blocks_seen": 22, "usage_input_tokens": 59330, "usage_output_tokens": 1298, "cost_usd": 0.1975, "harness_completed": true, "harness_errored": false }, "sympy__sympy-14248": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 33.0, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 89841, "usage_output_tokens": 907, "cost_usd": 0.2831, "harness_completed": true, "harness_errored": false }, "sympy__sympy-14531": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 39.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 36325, "usage_output_tokens": 1652, "cost_usd": 0.1338, "harness_completed": true, "harness_errored": false }, "sympy__sympy-15017": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 18.7, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 18751, "usage_output_tokens": 228, "cost_usd": 0.0597, "harness_completed": true, "harness_errored": false }, "sympy__sympy-15976": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 33.5, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 26113, "usage_output_tokens": 1200, "cost_usd": 0.0963, "harness_completed": true, "harness_errored": false }, "sympy__sympy-16450": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 28.1, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 34259, "usage_output_tokens": 853, "cost_usd": 0.1156, "harness_completed": true, "harness_errored": false }, "sympy__sympy-17655": { "resolved": true, "empty_patch": false, "agent_status": "ok", "agent_seconds": 21.6, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 25929, "usage_output_tokens": 380, "cost_usd": 0.0835, "harness_completed": true, "harness_errored": false }, "sympy__sympy-20916": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 42.2, "blocks_compressed": 0, "blocks_seen": 21, "usage_input_tokens": 90291, "usage_output_tokens": 1228, "cost_usd": 0.2893, "harness_completed": true, "harness_errored": false }, "sympy__sympy-21596": { "resolved": false, "empty_patch": false, "agent_status": "ok", "agent_seconds": 41.0, "blocks_compressed": 0, "blocks_seen": 13, "usage_input_tokens": 74215, "usage_output_tokens": 1440, "cost_usd": 0.2442, "harness_completed": true, "harness_errored": false } } } ], "paired_mcnemar": [ { "a": "full", "b": "distil_trunc500", "a_only": 20, "b_only": 2, "discordant": 22, "mcnemar_p": 0.0001 }, { "a": "full", "b": "llmlingua2", "a_only": 15, "b_only": 2, "discordant": 17, "mcnemar_p": 0.0023 }, { "a": "llmlingua2", "b": "distil_trunc500", "a_only": 7, "b_only": 2, "discordant": 9, "mcnemar_p": 0.1797 }, { "a": "full", "b": "distil_expand", "a_only": 2, "b_only": 4, "discordant": 6, "mcnemar_p": 0.6875 }, { "a": "distil_trunc500", "b": "distil_expand", "a_only": 2, "b_only": 22, "discordant": 24, "mcnemar_p": 0.0 }, { "a": "full", "b": "distil_gated", "a_only": 2, "b_only": 3, "discordant": 5, "mcnemar_p": 1.0 }, { "a": "distil_expand", "b": "distil_gated", "a_only": 5, "b_only": 4, "discordant": 9, "mcnemar_p": 1.0 } ], "total_cost_usd": 67.31, "wall_clock_seconds": null }