{
  "aliases": [],
  "authors": [],
  "details": {
    "benchmark_protocol": {
      "actions": [
        "GUI interactions",
        "command-line operations"
      ],
      "counts": [
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/README.md",
          "population": "documented MacAgentBench suite",
          "qualifier": "exact",
          "unit": "tasks",
          "value": 676
        },
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/README.md",
          "population": "documented application coverage",
          "qualifier": "exact",
          "unit": "applications",
          "value": 25
        }
      ],
      "evaluation": "Deterministic rule-based evaluators with intermediate checkpoints.",
      "evidence_urls": [
        "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/README.md",
        "https://export.arxiv.org/api/query?id_list=2605.06761%2C2605.07110%2C2605.07134%2C2605.09497%2C2605.11212%2C2605.12481%2C2605.12501%2C2605.12549%2C2605.13631%2C2605.14290%2C2605.14747%2C2605.15030%2C2605.15542%2C2605.15777%2C2605.16024%2C2605.16116%2C2605.16402%2C2605.17320%2C2605.18048%2C2605.18652%2C2605.18758%2C2605.19260%2C2605.19769%2C2605.21470%2C2605.25160%2C2605.25624%2C2605.25707%2C2605.26114%2C2605.27761%2C2605.28116%2C2605.28534%2C2605.28646%2C2605.28775%2C2605.29119%2C2605.29324%2C2605.29447%2C2605.29486%2C2605.29534%2C2605.29615%2C2606.06322%2C2606.09426%2C2606.11042%2C2606.22557%2C2606.23189%2C2606.24551%2C2608.17659%2C2609.22000%2C2609.34139&max_results=48"
      ],
      "focus": [
        "macOS task completion",
        "cross-application workflows"
      ],
      "notes": "The benchmark evaluates multiple agent frameworks and GUI/CLI workflows. Framework skills and interfaces affect the task setting; the inspected README does not define one universal observation representation.",
      "observations": [],
      "setting": "interactive"
    },
    "leaderboard_url": "https://jetastra.github.io/MacAgentBench/"
  },
  "environments": [
    "desktop",
    "macos"
  ],
  "evidence": [
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-16T06:35:59Z",
      "role": "primary",
      "sha256": "f1f8c0cd5d984e4026abea858b54f0f1427c018cad7e75322e744f1e070225a9",
      "supports": [
        "name",
        "kind",
        "summary",
        "environments",
        "topics",
        "scope",
        "relations",
        "details.benchmark_protocol",
        "details.leaderboard_url"
      ],
      "url": "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/README.md"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-16T06:32:03Z",
      "role": "primary",
      "sha256": "2adad07a9ec7189aea25e48c76fa28206eda97f8974d5745cadda9529103bb1a",
      "supports": [
        "details.benchmark_protocol",
        "relations"
      ],
      "url": "https://export.arxiv.org/api/query?id_list=2605.06761%2C2605.07110%2C2605.07134%2C2605.09497%2C2605.11212%2C2605.12481%2C2605.12501%2C2605.12549%2C2605.13631%2C2605.14290%2C2605.14747%2C2605.15030%2C2605.15542%2C2605.15777%2C2605.16024%2C2605.16116%2C2605.16402%2C2605.17320%2C2605.18048%2C2605.18652%2C2605.18758%2C2605.19260%2C2605.19769%2C2605.21470%2C2605.25160%2C2605.25624%2C2605.25707%2C2605.26114%2C2605.27761%2C2605.28116%2C2605.28534%2C2605.28646%2C2605.28775%2C2605.29119%2C2605.29324%2C2605.29447%2C2605.29486%2C2605.29534%2C2605.29615%2C2606.06322%2C2606.09426%2C2606.11042%2C2606.22557%2C2606.23189%2C2606.24551%2C2608.17659%2C2609.22000%2C2609.34139&max_results=48"
    },
    {
      "method": "license_header",
      "retrieved_at": "2026-10-16T06:41:59Z",
      "role": "primary",
      "sha256": "adf69997028988ba1eec34666afa119e007771dfc8e830c5dc50ad650896e897",
      "supports": [
        "licenses.code"
      ],
      "url": "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/LICENSE"
    }
  ],
  "id": "github:jetastra/macagentbench",
  "identifiers": {},
  "kind": "benchmark",
  "last_verified_at": "2026-10-16T06:35:59Z",
  "licenses": {
    "code": {
      "basis": "repository LICENSE header; not a weights/data license",
      "declared": "MIT",
      "retrieved_at": "2026-10-16T06:41:59Z",
      "url": "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/LICENSE"
    },
    "data": null,
    "document": null,
    "weights": null
  },
  "links": [
    {
      "rel": "primary",
      "url": "https://github.com/JetAstra/MacAgentBench"
    }
  ],
  "name": "MacAgentBench",
  "notes": "",
  "published_at": null,
  "relations": [
    {
      "evidence_url": "https://export.arxiv.org/api/query?id_list=2605.06761%2C2605.07110%2C2605.07134%2C2605.09497%2C2605.11212%2C2605.12481%2C2605.12501%2C2605.12549%2C2605.13631%2C2605.14290%2C2605.14747%2C2605.15030%2C2605.15542%2C2605.15777%2C2605.16024%2C2605.16116%2C2605.16402%2C2605.17320%2C2605.18048%2C2605.18652%2C2605.18758%2C2605.19260%2C2605.19769%2C2605.21470%2C2605.25160%2C2605.25624%2C2605.25707%2C2605.26114%2C2605.27761%2C2605.28116%2C2605.28534%2C2605.28646%2C2605.28775%2C2605.29119%2C2605.29324%2C2605.29447%2C2605.29486%2C2605.29534%2C2605.29615%2C2606.06322%2C2606.09426%2C2606.11042%2C2606.22557%2C2606.23189%2C2606.24551%2C2608.17659%2C2609.22000%2C2609.34139&max_results=48",
      "target_id": "arxiv:2606.22557",
      "type": "documented_by"
    }
  ],
  "review_status": "curated",
  "scope": "core",
  "summary": "676 macOS tasks across 25 applications, each run in its own Docker container and scored by deterministic rule-based checks with multiple checkpoints.",
  "topics": [
    "evaluation"
  ],
  "updated_at": null,
  "url": "https://github.com/JetAstra/MacAgentBench"
}