{"name":"Extcord benchmark","extcord_version":"0.5.1","generated_at":"2026-10-09T10:25:39+00:00","snapshots_captured_at":"2026-10-07T07:10:29+00:00","method":{"tokens":"Estimated at ~4 characters per token for every column, so ratios are comparable; exact counts vary by model.","baselines":"raw = the response of the obvious URL, read as is. reader = an HTML page converted to markdown, like typical reading tools. Each task is compared with its cheapest baseline among the URLs captured; a savvier agent might know a lighter endpoint for some of them.","extcord":"Request the agent writes plus the response it reads, through Extcord's HTTP API (MCP tool calls carry the same content).","correctness":"Every Extcord answer is checked against a truth computed independently from the same snapshots with plain Python.","not_measured":"Whether a model answers correctly after reading a large page. Exactness tasks have no token baseline: there Extcord costs tokens and buys correctness.","scope":"Developer lookups on PyPI, npm, crates.io and GitHub READMEs; general web pages aren't included yet."},"summary":{"tasks":18,"correct":18,"data_tasks":14,"cheapest_baseline_tokens":2521502,"extcord_terse_tokens":1899,"extcord_default_tokens":4994,"median_times_fewer":503.7,"median_times_fewer_default":119.3,"range_times_fewer":[3.8,13704.9],"median_saving_vs_cheapest":0.9954,"worst_task":"npm-latest","baselines_over_200k_tokens":["pypi-release-date","npm-count","pypi-count"],"exactness_tasks":4,"exactness_tokens_each":66,"session_overhead":{"mcp_tool_definitions":2221,"discover_query":630,"instructions":416}},"tasks":[{"id":"pypi-latest","category":"package facts","question":"What's the latest version of requests on PyPI?","baselines":[{"approach":"PyPI JSON API","url":"https://pypi.org/pypi/requests/json","mode":"raw","tokens":48241},{"approach":"PyPI page (HTML)","url":"https://pypi.org/project/requests/","mode":"raw","tokens":67429},{"approach":"PyPI page as markdown","url":"https://pypi.org/project/requests/","mode":"reader","tokens":7890}],"cheapest_baseline":{"approach":"PyPI page as markdown","url":"https://pypi.org/project/requests/","mode":"reader","tokens":7890},"extcord":{"how":"recipe","request_tokens":22,"response_tokens":269,"total":291},"extcord_terse":{"request_tokens":25,"response_tokens":80,"total":105},"answer":"2.34.2","truth":"2.34.2","correct":true,"saving_vs_cheapest":0.9867,"times_fewer":75.1},{"id":"pypi-python","category":"package facts","question":"Which Python versions does fastapi require?","baselines":[{"approach":"PyPI JSON API","url":"https://pypi.org/pypi/fastapi/json","mode":"raw","tokens":130678},{"approach":"PyPI page (HTML)","url":"https://pypi.org/project/fastapi/","mode":"raw","tokens":118238},{"approach":"PyPI page as markdown","url":"https://pypi.org/project/fastapi/","mode":"reader","tokens":15037}],"cheapest_baseline":{"approach":"PyPI page as markdown","url":"https://pypi.org/project/fastapi/","mode":"reader","tokens":15037},"extcord":{"how":"recipe","request_tokens":22,"response_tokens":325,"total":347},"extcord_terse":{"request_tokens":25,"response_tokens":135,"total":160},"answer":">=3.10","truth":">=3.10","correct":true,"saving_vs_cheapest":0.9894,"times_fewer":94.0},{"id":"pypi-release-date","category":"package facts","question":"What's numpy's latest version, and when was it released?","baselines":[{"approach":"PyPI JSON API","url":"https://pypi.org/pypi/numpy/json","mode":"raw","tokens":918229}],"cheapest_baseline":{"approach":"PyPI JSON API","url":"https://pypi.org/pypi/numpy/json","mode":"raw","tokens":918229},"extcord":{"how":"recipe","request_tokens":21,"response_tokens":330,"total":351},"extcord_terse":{"request_tokens":24,"response_tokens":140,"total":164},"answer":["2.5.3","2026-09-06T16:24:29.244371Z"],"truth":["2.5.3","2026-09-06T16:24:29.244371Z"],"correct":true,"saving_vs_cheapest":0.9998,"times_fewer":5599.0},{"id":"pypi-license","category":"package facts","question":"What license is Django under?","baselines":[{"approach":"PyPI JSON API","url":"https://pypi.org/pypi/django/json","mode":"raw","tokens":156481}],"cheapest_baseline":{"approach":"PyPI JSON API","url":"https://pypi.org/pypi/django/json","mode":"raw","tokens":156481},"extcord":{"how":"recipe","request_tokens":21,"response_tokens":341,"total":362},"extcord_terse":{"request_tokens":25,"response_tokens":152,"total":177},"answer":"BSD-3-Clause","truth":"BSD-3-Clause","correct":true,"saving_vs_cheapest":0.9989,"times_fewer":884.1},{"id":"npm-latest","category":"package facts","question":"Latest version of express on npm, and which Node.js versions it needs?","baselines":[{"approach":"npm registry document","url":"https://registry.npmjs.org/express","mode":"raw","tokens":202250},{"approach":"npm /latest document","url":"https://registry.npmjs.org/express/latest","mode":"raw","tokens":877}],"cheapest_baseline":{"approach":"npm /latest document","url":"https://registry.npmjs.org/express/latest","mode":"raw","tokens":877},"extcord":{"how":"recipe","request_tokens":21,"response_tokens":396,"total":417},"extcord_terse":{"request_tokens":24,"response_tokens":208,"total":232},"answer":["5.2.1",">= 18"],"truth":["5.2.1",">= 18"],"correct":true,"saving_vs_cheapest":0.7355,"times_fewer":3.8},{"id":"npm-license","category":"package facts","question":"Latest version and license of lodash?","baselines":[{"approach":"npm registry document","url":"https://registry.npmjs.org/lodash","mode":"raw","tokens":61920}],"cheapest_baseline":{"approach":"npm registry document","url":"https://registry.npmjs.org/lodash","mode":"raw","tokens":61920},"extcord":{"how":"pipeline","request_tokens":53,"response_tokens":264,"total":317},"extcord_terse":{"request_tokens":56,"response_tokens":12,"total":68},"answer":["4.18.1","MIT"],"truth":["4.18.1","MIT"],"correct":true,"saving_vs_cheapest":0.9989,"times_fewer":910.6},{"id":"crate-latest","category":"package facts","question":"Latest stable version of serde, and its total downloads?","baselines":[{"approach":"crates.io API","url":"https://crates.io/api/v1/crates/serde","mode":"raw","tokens":110248}],"cheapest_baseline":{"approach":"crates.io API","url":"https://crates.io/api/v1/crates/serde","mode":"raw","tokens":110248},"extcord":{"how":"recipe","request_tokens":20,"response_tokens":249,"total":269},"extcord_terse":{"request_tokens":23,"response_tokens":61,"total":84},"answer":["1.0.229",1492035683],"truth":["1.0.229",1492035683],"correct":true,"saving_vs_cheapest":0.9992,"times_fewer":1312.5},{"id":"crate-updated","category":"package facts","question":"Latest stable version of tokio, and when was the crate last updated?","baselines":[{"approach":"crates.io API","url":"https://crates.io/api/v1/crates/tokio","mode":"raw","tokens":111399}],"cheapest_baseline":{"approach":"crates.io API","url":"https://crates.io/api/v1/crates/tokio","mode":"raw","tokens":111399},"extcord":{"how":"recipe","request_tokens":20,"response_tokens":261,"total":281},"extcord_terse":{"request_tokens":23,"response_tokens":72,"total":95},"answer":["1.53.2","2026-10-03T11:18:32.487780Z"],"truth":["1.53.2","2026-10-03T11:18:32.487780Z"],"correct":true,"saving_vs_cheapest":0.9991,"times_fewer":1172.6},{"id":"npm-count","category":"counting","question":"How many versions of express have been published?","baselines":[{"approach":"npm registry document","url":"https://registry.npmjs.org/express","mode":"raw","tokens":202250}],"cheapest_baseline":{"approach":"npm registry document","url":"https://registry.npmjs.org/express","mode":"raw","tokens":202250},"extcord":{"how":"pipeline","request_tokens":55,"response_tokens":260,"total":315},"extcord_terse":{"request_tokens":59,"response_tokens":4,"total":63},"answer":289,"truth":289,"correct":true,"saving_vs_cheapest":0.9997,"times_fewer":3210.3},{"id":"pypi-count","category":"counting","question":"How many releases of numpy are on PyPI?","baselines":[{"approach":"PyPI JSON API","url":"https://pypi.org/pypi/numpy/json","mode":"raw","tokens":918229}],"cheapest_baseline":{"approach":"PyPI JSON API","url":"https://pypi.org/pypi/numpy/json","mode":"raw","tokens":918229},"extcord":{"how":"pipeline","request_tokens":60,"response_tokens":260,"total":320},"extcord_terse":{"request_tokens":63,"response_tokens":4,"total":67},"answer":151,"truth":151,"correct":true,"saving_vs_cheapest":0.9999,"times_fewer":13704.9},{"id":"readme-install","category":"reading","question":"How do I install requests, according to its README?","baselines":[{"approach":"README (markdown)","url":"https://raw.githubusercontent.com/psf/requests/main/README.md","mode":"raw","tokens":724}],"cheapest_baseline":{"approach":"README (markdown)","url":"https://raw.githubusercontent.com/psf/requests/main/README.md","mode":"raw","tokens":724},"extcord":{"how":"pipeline","request_tokens":62,"response_tokens":303,"total":365},"extcord_terse":{"request_tokens":65,"response_tokens":36,"total":101},"answer":"PyPI: ```console $ python -m pip install requests ``` Requests offici","truth":"$ python -m pip install requests","correct":true,"saving_vs_cheapest":0.8605,"times_fewer":7.2},{"id":"readme-outline","category":"reading","question":"What does fastapi's README cover? (its section headings)","baselines":[{"approach":"README (markdown)","url":"https://raw.githubusercontent.com/fastapi/fastapi/master/README.md","mode":"raw","tokens":5503}],"cheapest_baseline":{"approach":"README (markdown)","url":"https://raw.githubusercontent.com/fastapi/fastapi/master/README.md","mode":"raw","tokens":5503},"extcord":{"how":"recipe","request_tokens":29,"response_tokens":572,"total":601},"extcord_terse":{"request_tokens":33,"response_tokens":363,"total":396},"answer":["sponsors","keystone sponsor","gold sponsors","silver sponsors","opinions","fastapi mini documentary","typer, the fastapi of clis","requirements","installation","example","create it","run it","check it","interactive api docs","alternative api docs","example upgrade","interactive api docs upgrade","alternative api docs upgrade","recap","deploy your app (optional)","performance","dependencies","standard dependencies","without standard dependencies","without fastapi-cloud-cli","additional optional dependencies","license"],"truth":["sponsors","keystone sponsor","gold sponsors","silver sponsors","opinions","fastapi mini documentary","typer, the fastapi of clis","requirements","installation","example","create it","run it","check it","interactive api docs","alternative api docs","example upgrade","interactive api docs upgrade","alternative api docs upgrade","recap","deploy your app (optional)","performance","dependencies","standard dependencies","without standard dependencies","without fastapi-cloud-cli","additional optional dependencies","license"],"correct":true,"saving_vs_cheapest":0.928,"times_fewer":13.9},{"id":"page-summary","category":"reading","question":"What's the one-line summary on the requests PyPI page?","baselines":[{"approach":"PyPI page (HTML)","url":"https://pypi.org/project/requests/","mode":"raw","tokens":67429},{"approach":"PyPI page as markdown","url":"https://pypi.org/project/requests/","mode":"reader","tokens":7890}],"cheapest_baseline":{"approach":"PyPI page as markdown","url":"https://pypi.org/project/requests/","mode":"reader","tokens":7890},"extcord":{"how":"pipeline","request_tokens":52,"response_tokens":291,"total":343},"extcord_terse":{"request_tokens":55,"response_tokens":9,"total":64},"answer":"Python HTTP for Humans.","truth":"Python HTTP for Humans.","correct":true,"saving_vs_cheapest":0.9919,"times_fewer":123.3},{"id":"page-source-link","category":"reading","question":"Where is httpx's source code, according to its PyPI page?","baselines":[{"approach":"PyPI page (HTML)","url":"https://pypi.org/project/httpx/","mode":"raw","tokens":39443},{"approach":"PyPI page as markdown","url":"https://pypi.org/project/httpx/","mode":"reader","tokens":4825}],"cheapest_baseline":{"approach":"PyPI page as markdown","url":"https://pypi.org/project/httpx/","mode":"reader","tokens":4825},"extcord":{"how":"pipeline","request_tokens":69,"response_tokens":346,"total":415},"extcord_terse":{"request_tokens":72,"response_tokens":51,"total":123},"answer":["https://github.com/encode/httpx","https://github.com/encode/httpx/actions","https://github.com/encode/httpx/blob/master/CHANGELOG.md","https://github.com/encode/httpx/blob/master/LICENSE.md"],"truth":["https://github.com/encode/httpx","https://github.com/encode/httpx/actions","https://github.com/encode/httpx/blob/master/CHANGELOG.md","https://github.com/encode/httpx/blob/master/LICENSE.md"],"correct":true,"saving_vs_cheapest":0.9745,"times_fewer":39.2},{"id":"days-until","category":"exactness","question":"How many days from 2026-10-07 until 2027-01-01?","baselines":[],"cheapest_baseline":null,"extcord":{"how":"op","request_tokens":23,"response_tokens":156,"total":179},"extcord_terse":{"request_tokens":26,"response_tokens":30,"total":56},"answer":86,"truth":86,"correct":true},{"id":"business-days","category":"exactness","question":"What date is 10 business days after Wednesday 2026-10-07?","baselines":[],"cheapest_baseline":null,"extcord":{"how":"op","request_tokens":25,"response_tokens":170,"total":195},"extcord_terse":{"request_tokens":28,"response_tokens":44,"total":72},"answer":"2026-10-21","truth":"2026-10-21","correct":true},{"id":"money","category":"exactness","question":"Total for 3 items at $19.99 plus 8.25% tax, to the cent?","baselines":[],"cheapest_baseline":null,"extcord":{"how":"op","request_tokens":24,"response_tokens":118,"total":142},"extcord_terse":{"request_tokens":28,"response_tokens":4,"total":32},"answer":64.92,"truth":64.92,"correct":true},{"id":"timezones","category":"exactness","question":"9:00 in New York on 2026-11-02: what time is it in Tokyo? (US clocks changed the day before)","baselines":[],"cheapest_baseline":null,"extcord":{"how":"op","request_tokens":33,"response_tokens":193,"total":226},"extcord_terse":{"request_tokens":36,"response_tokens":66,"total":102},"answer":"2026-11-02T23:00:00+09:00","truth":"2026-11-02T23:00:00+09:00","correct":true}]}