{
  "aliases": [
    "github:reacher-z/clawbench",
    "reacher-z/ClawBench",
    "https://github.com/reacher-z/ClawBench"
  ],
  "authors": [],
  "details": {
    "benchmark_protocol": {
      "actions": [
        "harness-defined browser actions such as navigating, clicking and filling forms"
      ],
      "counts": [
        {
          "basis": "computed",
          "evidence_url": "https://api.github.com/repos/TIGER-AI-Lab/ClawBench/git/trees/d12a2dda577f2c943a74d833e8253860eda81eaf?recursive=1",
          "population": "V1 corpus shipped in test-cases/v1 (task.json files)",
          "qualifier": "exact",
          "unit": "tasks",
          "value": 152
        },
        {
          "basis": "computed",
          "evidence_url": "https://api.github.com/repos/TIGER-AI-Lab/ClawBench/git/trees/d12a2dda577f2c943a74d833e8253860eda81eaf?recursive=1",
          "population": "V2 corpus shipped in test-cases/v2 (task.json files)",
          "qualifier": "exact",
          "unit": "tasks",
          "value": 129
        },
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/eval/scoring.md",
          "population": "V1 corpus as published in the paper and used as the scoring denominator",
          "qualifier": "exact",
          "unit": "tasks",
          "value": 153
        },
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/eval/scoring.md",
          "population": "V2 corpus as published and used as the scoring denominator",
          "qualifier": "exact",
          "unit": "tasks",
          "value": 130
        },
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/HEAD/README.md",
          "population": "shipped V1 corpus",
          "qualifier": "exact",
          "unit": "live websites",
          "value": 143
        },
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/HEAD/README.md",
          "population": "shipped V2 corpus",
          "qualifier": "exact",
          "unit": "live websites",
          "value": 63
        }
      ],
      "evaluation": "Two stages per task run: a request interceptor inside the container blocks the final HTTP request and marks the run intercepted when its URL regex and method match the task's eval_schema; an LLM judge (default deepseek-v4-pro at temperature 0) then checks the intercepted body against the instruction. A pass needs both. The original V1 paper rubric instead compares agent and human-reference trajectories.",
      "evidence_urls": [
        "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/HEAD/README.md",
        "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/eval/scoring.md",
        "https://api.github.com/repos/TIGER-AI-Lab/ClawBench/git/trees/d12a2dda577f2c943a74d833e8253860eda81eaf?recursive=1"
      ],
      "focus": [
        "everyday online tasks on production websites",
        "multi-step, write-heavy form workflows"
      ],
      "notes": "The README says two ASPCA tasks were removed after publication, so the shipped corpora hold 152 and 129 tasks while scoring.md still divides by 153 and 130. Live sites change over time, and the README asks for comparisons under the same corpus revision, harness and rubric. Agents run through selectable harnesses (for example Hermes and OpenClaw), so observations and actions depend on the harness.",
      "observations": [],
      "setting": "interactive"
    },
    "leaderboard_url": "https://huggingface.co/spaces/TIGER-Lab/ClawBench",
    "repository_revision": "07eb50a3d4c5a799f5ad4eebd57c67c10417787f",
    "repository_tree_sha": "d12a2dda577f2c943a74d833e8253860eda81eaf"
  },
  "environments": [
    "web"
  ],
  "evidence": [
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-18T11:53:46Z",
      "role": "primary",
      "sha256": "61bad2a8e6ab56df159237d75f1081e1246b294183b31ef7abe47193ac26b541",
      "supports": [
        "name",
        "kind",
        "summary",
        "environments",
        "topics",
        "scope",
        "notes",
        "relations",
        "details"
      ],
      "url": "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/HEAD/README.md"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-18T11:54:35Z",
      "role": "primary",
      "sha256": "144a65c3dd92c7437e4ce065bed7b16e4c33496ea395d2bb1ecbd74132da29ce",
      "supports": [
        "details.repository_revision",
        "details.repository_tree_sha",
        "details.benchmark_protocol"
      ],
      "url": "https://api.github.com/repos/TIGER-AI-Lab/ClawBench/git/trees/d12a2dda577f2c943a74d833e8253860eda81eaf?recursive=1"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-18T11:54:36Z",
      "role": "primary",
      "sha256": "8d8ade83a355e10800591435de5aa111e8d8297374b025ff88804358ee80b71a",
      "supports": [
        "details.benchmark_protocol"
      ],
      "url": "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/07eb50a3d4c5a799f5ad4eebd57c67c10417787f/eval/scoring.md"
    },
    {
      "method": "license_header",
      "retrieved_at": "2026-10-18T11:55:00Z",
      "role": "primary",
      "sha256": "cb861e529412e9eec01498bdf4b74c5ee4f29139737a1193e39b24004019e606",
      "supports": [
        "licenses.code"
      ],
      "url": "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/HEAD/LICENSE"
    },
    {
      "method": "github_repository_identity",
      "retrieved_at": "2026-10-18T11:54:36Z",
      "role": "primary",
      "sha256": "4b80afa11f3998830090c9c186afc753f261cc05abbcd4d906400f757f41e4da",
      "supports": [
        "aliases"
      ],
      "url": "https://api.github.com/repos/reacher-z/ClawBench"
    },
    {
      "method": "github_repository_identity",
      "retrieved_at": "2026-10-18T11:54:37Z",
      "role": "primary",
      "sha256": "4b80afa11f3998830090c9c186afc753f261cc05abbcd4d906400f757f41e4da",
      "supports": [
        "aliases"
      ],
      "url": "https://api.github.com/repos/TIGER-AI-Lab/ClawBench"
    }
  ],
  "id": "github:tiger-ai-lab/clawbench",
  "identifiers": {},
  "kind": "benchmark",
  "last_verified_at": "2026-10-18T11:53:46Z",
  "licenses": {
    "code": {
      "basis": "repository LICENSE header; not a weights/data license",
      "declared": "Apache-2.0",
      "retrieved_at": "2026-10-18T11:55:00Z",
      "url": "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/HEAD/LICENSE"
    },
    "data": null,
    "document": null,
    "weights": null
  },
  "links": [
    {
      "rel": "primary",
      "url": "https://github.com/TIGER-AI-Lab/ClawBench"
    }
  ],
  "name": "ClawBench",
  "notes": "Renamed from reacher-z/ClawBench; both API names return repository 1206519531 (TIGER-AI-Lab/ClawBench), which is not a fork and not archived. Counts and scoring rules were read at commit 07eb50a3. The tree also holds v1-lite, claw-eval and environmental-observation task folders, which are not counted here. Task definitions and execution traces are also published as Hugging Face datasets (NAIL-Group/ClawBench, NAIL-Group/ClawBenchV1Trace, TIGER-Lab/ClawBenchV2Trace), whose cards and licenses were not reviewed for this record. Apache-2.0 is recorded for repository code only. Nothing was installed or run.",
  "published_at": null,
  "relations": [
    {
      "evidence_url": "https://raw.githubusercontent.com/TIGER-AI-Lab/ClawBench/HEAD/README.md",
      "target_id": "arxiv:2604.08523",
      "type": "documented_by"
    }
  ],
  "review_status": "curated",
  "scope": "core",
  "summary": "Everyday-task benchmark on live websites that runs browser agents in containerized Chromium, blocks each final submission request for scoring by URL match and an LLM judge, and records five layers of run evidence.",
  "topics": [
    "evaluation",
    "web-navigation",
    "workflow",
    "observability"
  ],
  "updated_at": null,
  "url": "https://github.com/TIGER-AI-Lab/ClawBench"
}