{
  "aliases": [],
  "authors": [],
  "details": {
    "benchmark_protocol": {
      "actions": [
        "agent-specific Android actions"
      ],
      "counts": [
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md",
          "population": "complete README task collection",
          "qualifier": "exact",
          "unit": "tasks",
          "value": 340
        },
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md",
          "population": "single-app subset",
          "qualifier": "exact",
          "unit": "tasks",
          "value": 300
        },
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md",
          "population": "cross-app subset",
          "qualifier": "exact",
          "unit": "tasks",
          "value": 40
        },
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md",
          "population": "README total including third-party, Google, and system apps",
          "qualifier": "exact",
          "unit": "applications",
          "value": 66
        }
      ],
      "evaluation": "A multimodal evaluator and annotated key-component matching assess completion; the framework also reports resource-use metrics.",
      "evidence_urls": [
        "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md",
        "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/Documentation.md"
      ],
      "focus": [
        "mobile task completion",
        "resource use",
        "cross-app workflows"
      ],
      "notes": "The two task subsets partition the 340 tasks. The integrated agent and evaluator modes must accompany results. The preconfigured emulator snapshot is still marked coming soon in the inspected documentation.",
      "observations": [
        "screenshot trajectories"
      ],
      "setting": "interactive"
    }
  },
  "environments": [
    "android"
  ],
  "evidence": [
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-16T06:35:58Z",
      "role": "primary",
      "sha256": "5294a67728072a3658cb2073bdf9a084fe3c00d87a848923dff68adb35213072",
      "supports": [
        "name",
        "kind",
        "summary",
        "environments",
        "topics",
        "scope",
        "relations",
        "details"
      ],
      "url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-16T06:35:58Z",
      "role": "primary",
      "sha256": "5294a67728072a3658cb2073bdf9a084fe3c00d87a848923dff68adb35213072",
      "supports": [
        "details.benchmark_protocol"
      ],
      "url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-17T02:03:23Z",
      "role": "primary",
      "sha256": "a55290d62f8823097c73d74bac4eb08853a3caff724382e60a6a9364e7ced987",
      "supports": [
        "details.benchmark_protocol"
      ],
      "url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/Documentation.md"
    },
    {
      "method": "license_header",
      "retrieved_at": "2026-10-16T06:41:58Z",
      "role": "primary",
      "sha256": "6962bfc8ee16c6e985e4fb680b8fe46d5b34d30f7cb8a1e61fd4c55c93120ddf",
      "supports": [
        "licenses.code"
      ],
      "url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/LICENSE"
    }
  ],
  "id": "github:ai-agents-2030/spa-bench",
  "identifiers": {},
  "kind": "benchmark",
  "last_verified_at": "2026-10-16T06:35:58Z",
  "licenses": {
    "code": {
      "basis": "repository LICENSE header; not a weights/data license",
      "declared": "MIT",
      "retrieved_at": "2026-10-16T06:41:58Z",
      "url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/LICENSE"
    },
    "data": null,
    "document": null,
    "weights": null
  },
  "links": [
    {
      "rel": "primary",
      "url": "https://github.com/ai-agents-2030/SPA-Bench"
    }
  ],
  "name": "SPA-Bench",
  "notes": "",
  "published_at": null,
  "relations": [
    {
      "evidence_url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md",
      "target_id": "arxiv:2410.15164",
      "type": "documented_by"
    }
  ],
  "review_status": "curated",
  "scope": "core",
  "summary": "340 single-app and cross-app tasks over 66 Chinese and English Android apps, with plug-in support for 11 smartphone agents and automatic coarse-to-fine success detection.",
  "topics": [
    "evaluation",
    "mobile-navigation"
  ],
  "updated_at": null,
  "url": "https://github.com/ai-agents-2030/SPA-Bench"
}