{
  "aliases": [],
  "authors": [],
  "details": {
    "benchmark_protocol": {
      "actions": [
        "OSWorld/PyAutoGUI desktop interaction"
      ],
      "counts": [
        {
          "basis": "reported",
          "evidence_url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/README.md",
          "population": "reported benchmark total; not a recount of unique released task IDs",
          "qualifier": "exact",
          "unit": "tasks as reported by the paper/README",
          "value": 150
        },
        {
          "basis": "computed",
          "evidence_url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misuse.json",
          "population": "released misuse task-ID list; before injection/goal expansion",
          "qualifier": "exact",
          "unit": "task IDs",
          "value": 50
        },
        {
          "basis": "computed",
          "evidence_url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_injection.json",
          "population": "released injection task-ID list; before injection/goal expansion",
          "qualifier": "exact",
          "unit": "task IDs",
          "value": 10
        },
        {
          "basis": "computed",
          "evidence_url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misbehavior.json",
          "population": "released misbehavior task-ID list; before injection/goal expansion",
          "qualifier": "exact",
          "unit": "task IDs",
          "value": 50
        }
      ],
      "evaluation": "Automated LLM judge scores both task accuracy and safety from saved execution traces. Keep judge model/prompt version and harm category with a result; the scripts separately compute predicted metrics. This is an interactive desktop safety benchmark, not a single offline grounding score.",
      "evidence_urls": [
        "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/README.md",
        "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/run.py",
        "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misuse.json",
        "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_injection.json",
        "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misbehavior.json"
      ],
      "focus": [
        "deliberate misuse",
        "prompt-injection safety",
        "unintended harmful behavior during normal tasks"
      ],
      "notes": "README/paper reports 150 tasks, whereas retained misuse/injection/misbehavior lists contain 50/10/50 IDs. run.py expands injection vectors/goals, so the released ID inventory is not a direct recount of paper settings; the complete expanded total was not reconstructed. Judge agreement F1 values in the README measure agreement with human labels, not agent success. Experiments use OSWorld desktop apps, with VMware documented as tested and screenshot/a11y/SOM observation choices. Code Apache-2.0 does not establish VM/app/data rights. Publisher asks that the benchmark not be used directly as agent training data.",
      "observations": [
        "screenshot",
        "accessibility tree",
        "screenshot plus accessibility tree",
        "set-of-marks"
      ],
      "setting": "interactive"
    },
    "evaluation_type": "automated judgment of accuracy and safety",
    "metrics": [
      "task completion",
      "safety"
    ],
    "task_count": 150,
    "task_unit": "tasks reported by the paper/README; not a recount of released task IDs"
  },
  "environments": [
    "desktop"
  ],
  "evidence": [
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-16T09:36:33Z",
      "role": "primary",
      "sha256": "d001158f58ac37b01c8470224db1567a52feb8010877305f7966637cf7400706",
      "supports": [
        "name",
        "kind",
        "summary",
        "environments",
        "topics",
        "scope",
        "relations",
        "details",
        "details.benchmark_protocol",
        "notes"
      ],
      "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/README.md"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-17T14:23:14Z",
      "role": "primary",
      "sha256": "86dd59a252fd654fc3244268a75de37b2c7f97e16f6b1de8c4cf885458a422f7",
      "supports": [
        "details.benchmark_protocol",
        "notes"
      ],
      "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/run.py"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-17T14:22:04Z",
      "role": "primary",
      "sha256": "8ff6092a51cc924c5225ace01c011c6481a2461eb0a47b7360eddaf6619b898c",
      "supports": [
        "details.benchmark_protocol",
        "notes"
      ],
      "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misuse.json"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-17T14:22:04Z",
      "role": "primary",
      "sha256": "b4c451236bb4ff36eb7e790dfa8cb517bffb245602bc441080314b13c8aba54f",
      "supports": [
        "details.benchmark_protocol",
        "notes"
      ],
      "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_injection.json"
    },
    {
      "method": "editorial_primary_review",
      "retrieved_at": "2026-10-17T14:22:05Z",
      "role": "primary",
      "sha256": "8eb6bade72696c3c0b53160a05b006a773c9e553ab5e401418b54d9acc98c0f2",
      "supports": [
        "details.benchmark_protocol",
        "notes"
      ],
      "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misbehavior.json"
    },
    {
      "method": "license_header",
      "retrieved_at": "2026-10-16T09:39:34Z",
      "role": "primary",
      "sha256": "7c3ab1eeeb073d92238ecb65ffaebe58ab5922929b115a7c0b64eb4f9a9afc2f",
      "supports": [
        "licenses.code"
      ],
      "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/LICENSE"
    }
  ],
  "id": "github:tml-epfl/os-harm",
  "identifiers": {},
  "kind": "benchmark",
  "last_verified_at": "2026-10-16T09:36:33Z",
  "licenses": {
    "code": {
      "basis": "repository LICENSE header; not a weights/data license",
      "declared": "Apache-2.0",
      "retrieved_at": "2026-10-16T09:39:34Z",
      "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/LICENSE"
    },
    "data": null,
    "document": null,
    "weights": null
  },
  "links": [
    {
      "rel": "primary",
      "url": "https://github.com/tml-epfl/os-harm"
    }
  ],
  "name": "OS-Harm",
  "notes": "The retained task-ID manifests have 50 misuse, 10 injection and 50 misbehavior IDs. Prompt-injection execution expands vectors/goals; the reported 150-task total must not be read as 150 unique released IDs.",
  "published_at": null,
  "relations": [
    {
      "evidence_url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/README.md",
      "target_id": "arxiv:2506.14866",
      "type": "documented_by"
    }
  ],
  "review_status": "curated",
  "scope": "core",
  "summary": "Benchmark for safety of computer-use agents across misuse, prompt injection, and unintended model behavior.",
  "topics": [
    "safety-security",
    "evaluation"
  ],
  "updated_at": null,
  "url": "https://github.com/tml-epfl/os-harm"
}