{
  "title": "ARE-bench: Benchmarking Agents' Research Ability in Real-World Researcher Sessions and Workspaces",
  "affiliations": [
    "Shanghai Artificial Intelligence Laboratory",
    "Fudan University",
    "East China Normal University"
  ],
  "authors": {
    "core": [
      "Jiahang Lin†",
      "Wanying Ren†",
      "Wanghan Xu†",
      "Huayu Sha",
      "Yang Shu",
      "Zhonghang Lu",
      "Chengjun Pan",
      "Yi Dong"
    ],
    "contributors": [
      "Yuankai Ying",
      "Hao Peng",
      "Chunchun Ma",
      "Hanyu Zhang",
      "Yiding Wang",
      "Yuyang Hu",
      "Huijie Lv",
      "Shichun Liu",
      "Shihan Dou",
      "Honglin Guo",
      "Pujun Zheng",
      "Zhe Sun",
      "Jiahao Chen",
      "Zhixiong Yang"
    ],
    "advisors": [
      "Dingbo Yuan",
      "Yicheng Zou",
      "Zhecan Wang",
      "Guoxiu He",
      "Xiaoyu Xing",
      "Qi Zhang✉",
      "Tao Gui✉",
      "Xuanjing Huang"
    ],
    "notes": "† Project leaders · ✉ Corresponding authors"
  },
  "links": {
    "paper": "https://arxiv.org/abs/2611.14364",
    "code": null,
    "data": null
  },
  "stats": [
    {
      "label": "Tasks",
      "value": 75
    },
    {
      "label": "Checklist items",
      "value": 1173,
      "comma": true
    },
    {
      "label": "Research stages",
      "value": 7
    },
    {
      "label": "Requirements",
      "value": 33
    },
    {
      "label": "Sub-requirements",
      "value": 150
    },
    {
      "label": "Earlier requests in Background (avg.)",
      "value": 10.3,
      "dec": 1
    },
    {
      "label": "Workspace files at cut point (avg.)",
      "value": 41.1,
      "dec": 1
    },
    {
      "label": "Tasks with agent-written files",
      "value": 65.3,
      "dec": 1,
      "suffix": "%"
    }
  ],
  "findings": [
    {
      "rq": "RQ1",
      "title": "Passing most checklist items does not translate into task-level success.",
      "body": "All seven models pass 79.7% to 92.7% of checklist items, but their ISR ranges from 25.8% to 66.2%. GPT-6.1-Sol passes 92.7% of items and still fails 33.8% of tasks; a single miss leaves a task unusable."
    },
    {
      "rq": "RQ2",
      "title": "Correctness is the binding constraint on strict success.",
      "body": "Correctness ISR spans 31.1% to 71.1%, far below completeness (72.0% to 87.6%) and constraint compliance (79.1% to 93.8%). Agents deliver the requested artifacts in the requested form but get some of the content wrong."
    },
    {
      "rq": "RQ3",
      "title": "Design and Execution are among the three hardest stages for every model; Data & Env. is the easiest.",
      "body": "Averaged over seven models, stage ISR falls from 86.4% on Data & Env. and 71.3% on Code to 59.5% on Design and 57.4% on Execution. These two stages also separate models the most, by 35.8 and 37.4 points."
    },
    {
      "rq": "RQ3",
      "title": "Code skill does not carry over to Execution and Writing.",
      "body": "Kimi-K3 ranks third on Code at 74.0% but records the lowest Writing ISR (47.3%) and the second-lowest Execution ISR (47.5%)."
    },
    {
      "rq": "Position",
      "title": "Tasks from the first segment of a session score lower; later segments show no clear trend.",
      "body": "Across 50 tasks cut from five segments of 10 sessions, GLM-5.3 reaches 87.0% CSR in the first segment against 94.7% in the remaining four. Early tasks still require turning a topic into an executable research plan."
    },
    {
      "rq": "RQ4",
      "title": "Higher token spend does not buy strict success.",
      "body": "DeepSeek-V4.1-Flash uses 3.1 times the input tokens of GPT-6.1-Sol for 39.6% ISR. GPT-6.1-Sol costs $1.66 per rollout; Claude Opus 5.5 costs 2.4 times as much for 56.9%."
    }
  ],
  "figure1": {
    "source": "Paper Figure 2: one real research session",
    "total_turns": 42,
    "cut_turn": 31,
    "turns": [
      {
        "t": 1,
        "text": "Restructure the original draft."
      },
      {
        "t": 7,
        "text": "Check similar papers."
      },
      {
        "t": 9,
        "text": "The reviews are back."
      },
      {
        "t": 18,
        "text": "Rewrite the code from scratch."
      },
      {
        "t": 22,
        "text": "Recompute anomaly 1."
      },
      {
        "t": 27,
        "text": "Is it a hard flaw of our method?"
      },
      {
        "t": 31,
        "text": "Rebuild the paper's storyline. The structure still carries residue from earlier work.",
        "target": true
      }
    ],
    "workspace": [
      {
        "path": "uploads/review_comments_…_3fa1c2d8.pdf",
        "tag": "duplicate"
      },
      {
        "path": "uploads/review_comments_…_7c41e0b2.pdf",
        "tag": "duplicate"
      },
      {
        "path": "wisdoc_pdf_parse_result/",
        "tag": "parsed reviews"
      },
      {
        "path": "artifacts/new_paper_plan.md … _v4.md",
        "tag": "superseded"
      },
      {
        "path": "artifacts/new_paper_plan_v5.md",
        "tag": "current"
      },
      {
        "path": "artifacts/section3_final.md",
        "tag": "stale"
      },
      {
        "path": "artifacts/section3_revised.md",
        "tag": "current"
      },
      {
        "path": "tic-experiments/exp01 … exp09",
        "tag": "paused"
      }
    ]
  },
  "bibtex": "@misc{lin2026arebench,\n  title         = {ARE-bench: Benchmarking Agents' Research Ability in Real-World Researcher Sessions and Workspaces},\n  author        = {Lin, Jiahang and Ren, Wanying and Xu, Wanghan and Sha, Huayu and Shu, Yang and Lu, Zhonghang and Pan, Chengjun and Dong, Yi and Liu, Shichun and Hu, Yuyang and Dou, Shihan and Guo, Honglin and Lv, Huijie and Zheng, Pujun and Sun, Zhe and Ying, Yuankai and Wang, Yiding and Chen, Jiahao and Yang, Zhixiong and Yuan, Dingbo and Zou, Yicheng and Wang, Zhecan and He, Guoxiu and Xing, Xiaoyu and Zhang, Qi and Gui, Tao and Huang, Xuanjing},\n  year          = {2026},\n  eprint        = {2611.14364},\n  archivePrefix = {arXiv},\n  primaryClass  = {cs.AI},\n  url           = {https://arxiv.org/abs/2611.14364}\n}"
}
