{
  "aliases": [],
  "authors": [],
  "details": {
    "availability": "Public benchmark code and public Zenodo dataset release metadata.",
    "benchmark_protocol": {
      "actions": [
        "generate procedures",
        "segment demonstrations",
        "answer workflow questions",
        "validate and rank procedures"
      ],
      "counts": [
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/HazyResearch/wonderbread/HEAD/README.md",
          "population": "WONDERBREAD BPM benchmark",
          "qualifier": "exact",
          "unit": "task types",
          "value": 6
        },
        {
          "basis": "reported",
          "evidence_url": "https://zenodo.org/api/records/14094162",
          "population": "full dataset release 14094162",
          "qualifier": "exact",
          "unit": "human demonstrations",
          "value": 2928
        },
        {
          "basis": "reported",
          "evidence_url": "https://zenodo.org/api/records/14094162",
          "population": "full dataset release 14094162",
          "qualifier": "exact",
          "unit": "web navigation workflows",
          "value": 598
        },
        {
          "basis": "reported",
          "evidence_url": "https://zenodo.org/api/records/14094162",
          "population": "gold subset used for the original paper evaluations",
          "qualifier": "exact",
          "unit": "human demonstrations",
          "value": 724
        }
      ],
      "evaluation": "Task-specific scripts evaluate six workflow-understanding task types; model outputs are not treated as successful interactive execution.",
      "evidence_urls": [
        "https://raw.githubusercontent.com/HazyResearch/wonderbread/HEAD/README.md",
        "https://zenodo.org/api/records/14094162"
      ],
      "focus": [
        "workflow documentation",
        "workflow quality assessment",
        "workflow knowledge transfer"
      ],
      "notes": "Counts are publisher-reported. The gold subset covers 162 workflows and differs from the full collection; scores require the exact task and subset.",
      "observations": [
        "screen recordings",
        "action traces",
        "screenshots",
        "written standard operating procedures"
      ],
      "setting": "offline"
    },
    "dataset_release": "10.5281/zenodo.14094162",
    "license_review": {
      "data": {
        "evidence_urls": [
          "https://zenodo.org/api/records/14094162"
        ],
        "reason": "The linked Zenodo dataset release declares CC-BY-4.0; this does not assign a code license to the repository."
      }
    },
    "repository_revision": "ed052c67aeada04167cdfe92ff8de454aa94627a"
  },
  "environments": [
    "web"
  ],
  "evidence": [
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-17T09:12:11Z",
      "role": "primary",
      "sha256": "1e4308d59a22b39543c28b745a58d3d7c47be1ab9df9a4c5c0fe868a810350b2",
      "supports": [
        "name",
        "kind",
        "summary",
        "environments",
        "topics",
        "scope",
        "notes",
        "relations",
        "details"
      ],
      "url": "https://raw.githubusercontent.com/HazyResearch/wonderbread/HEAD/README.md"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-17T09:12:11Z",
      "role": "primary",
      "sha256": "5ba27e24455b36b52bf65b3c9f589e760f97a576bb01d622d89836c5bb53b38c",
      "supports": [
        "identity"
      ],
      "url": "https://api.github.com/repos/HazyResearch/wonderbread"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-17T09:14:04Z",
      "role": "primary",
      "sha256": "accb518a5362948cf96e9f0b684fb6fcc9b26e43a691016ae70a3283c0e1e2c2",
      "supports": [
        "release_status",
        "licenses.code"
      ],
      "url": "https://api.github.com/repos/HazyResearch/wonderbread/git/trees/main?recursive=1"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-17T09:15:34Z",
      "role": "primary",
      "sha256": "68b4b952c1ab90484d457b392de1d0c95a24f6cc1674c82125a6bd26f1cb1016",
      "supports": [
        "release_status",
        "licenses.data",
        "details.benchmark_protocol"
      ],
      "url": "https://zenodo.org/api/records/14094162"
    }
  ],
  "id": "github:hazyresearch/wonderbread",
  "identifiers": {},
  "kind": "benchmark",
  "last_verified_at": "2026-10-17T09:12:11Z",
  "licenses": {
    "code": null,
    "data": {
      "basis": "The linked Zenodo dataset release declares CC-BY-4.0; this does not assign a code license to the repository.",
      "declared": "cc-by-4.0",
      "retrieved_at": "2026-10-17T09:15:34Z",
      "url": "https://zenodo.org/api/records/14094162"
    },
    "document": null,
    "weights": null
  },
  "links": [
    {
      "rel": "primary",
      "url": "https://github.com/HazyResearch/wonderbread"
    }
  ],
  "name": "WONDERBREAD",
  "notes": "The repository contains evaluation and collection code; the linked Zenodo release exposes the full, gold and debug demonstration archives and metadata. This measures workflow understanding rather than interactive agent task completion. Zenodo declares CC-BY-4.0 for that dataset; no repository-level code license was found. Archives and human demonstration records were not downloaded.",
  "published_at": null,
  "relations": [
    {
      "evidence_url": "https://raw.githubusercontent.com/HazyResearch/wonderbread/HEAD/README.md",
      "target_id": "arxiv:2406.13264",
      "type": "documented_by"
    }
  ],
  "review_status": "curated",
  "scope": "core",
  "summary": "An offline benchmark of workflow documentation, quality assessment and knowledge transfer using human web-navigation demonstrations, action traces and written procedures.",
  "topics": [
    "evaluation",
    "workflow",
    "video",
    "screen-understanding"
  ],
  "updated_at": null,
  "url": "https://github.com/HazyResearch/wonderbread"
}