{ "schema_version": 1, "kind": "benchmark", "profile_count": 61, "curated_resource_count": 89, "unit": "resource profile; count observations retain their own units and populations", "limitations": "Source-reviewed descriptions and reported or computed inventory counts. No performance ranking, independent reproduction, or global-completeness claim.", "records": [ { "id": "github:mcgill-nlp/agent-reward-bench", "name": "AgentRewardBench", "url": "https://github.com/McGill-NLP/agent-reward-bench", "environments": [ "web" ], "profile": { "actions": [ "judge verdicts" ], "counts": [ { "basis": "reported", "evidence_url": "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=700&max_results=100&sortBy=submittedDate&sortOrder=descending", "population": "paper-reported expert-reviewed set", "qualifier": "exact", "unit": "trajectories", "value": 1302 } ], "evaluation": "Compare automatic judgments with expert answers on task success, side effects and repetitiveness.", "evidence_urls": [ "https://raw.githubusercontent.com/McGill-NLP/agent-reward-bench/HEAD/README.md", "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=700&max_results=100&sortBy=submittedDate&sortOrder=descending" ], "focus": [ "automatic evaluator agreement", "task success", "side effects", "repetition" ], "notes": "Counts refer to the paper study; current Hub package files are not converted into trajectory totals.", "observations": [ "web trajectories", "screenshots", "actions" ], "setting": "offline" }, "licenses": { "code": null, "data": { "basis": "Publisher dataset-card Terms of Use, including third-party restrictions; no standard SPDX or unrestricted permission is asserted.", "declared": "Custom AgentRewardBench Terms of Use", "retrieved_at": "2026-10-17T04:03:20Z", "url": "https://huggingface.co/datasets/McGill-NLP/agent-reward-bench/raw/main/README.md" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T03:59:54Z", "role": "primary", "sha256": "fb753e41c3eb19be172e9e7fccbc6bcadf44288507f82b190ca99e64144294b6", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/McGill-NLP/agent-reward-bench/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:36:54Z", "role": "primary", "sha256": "4953a8e0f029016aef668bce98cec4d2873a7ae49c1fe33c4311fa5eb8d24089", "supports": [ "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=700&max_results=100&sortBy=submittedDate&sortOrder=descending" } ] }, { "id": "github:androidarenaagent/androidarena", "name": "AndroidArena", "url": "https://github.com/AndroidArenaAgent/AndroidArena", "environments": [ "android" ], "profile": { "actions": [ "app launch/stop", "click/double-click/long-click", "text input", "swipe/drag and Android navigation", "finish" ], "counts": [], "evaluation": "The released evaluator compares trajectories with reference actions using longest-common-subsequence-based task reward and completion. Its reversed redundancy metric is the reference/predicted action-length ratio (with 1e-6 denominator smoothing), multiplied by the LLM success flag. LLM success judgments and instruction-constraint checks are separate metrics. Full reference-dependent evaluation coverage cannot be assumed because complete annotated action sequences are not released.", "evidence_urls": [ "https://raw.githubusercontent.com/AndroidArenaAgent/AndroidArena/HEAD/README.md", "https://raw.githubusercontent.com/AndroidArenaAgent/AndroidArena/HEAD/android_env2/README.md", "https://api.github.com/repos/AndroidArenaAgent/AndroidArena/git/trees/main?recursive=1", "https://raw.githubusercontent.com/AndroidArenaAgent/AndroidArena/HEAD/android_env2/observation.py", "https://raw.githubusercontent.com/AndroidArenaAgent/AndroidArena/HEAD/android_env2/actions.py", "https://raw.githubusercontent.com/AndroidArenaAgent/AndroidArena/HEAD/run_evaluator.py" ], "focus": [ "multi-step Android tasks", "cross-app workflows and instruction constraints" ], "notes": "README states that task instructions are released but annotated action sequences, apart from the calendar example, will come later. Environment setup targets Android API 33 with uiautomator2 and manual Google-account setup. Public task YAML and evaluator code exist, but benchmark execution and full reference coverage were not verified. No task total is derived from app YAML file counts. Code and data licenses remain unknown.", "observations": [ "UI hierarchy text", "screenshots", "combined text/image observations" ], "setting": "interactive" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "206bd0dc5220cf754cffc97767cce6a3cd27660ef050a569366bef2331241c60", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/AndroidArenaAgent/AndroidArena/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:10Z", "role": "primary", "sha256": "a1e7dc0c5facae030c2cf19aad6511996e7e1198d8d6619268d10c29a1b57bcf", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/AndroidArenaAgent/AndroidArena/HEAD/android_env2/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:14:05Z", "role": "primary", "sha256": "edc03388b67143644db4a412e4c888f8c796068daff29f63615f0b6a3a04262a", "supports": [ "details.benchmark_protocol" ], "url": "https://api.github.com/repos/AndroidArenaAgent/AndroidArena/git/trees/main?recursive=1" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:24:52Z", "role": "primary", "sha256": "cfa9d272c85398f54c18d6e7445d680009e8c16d128962235d899592d3094c2f", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/AndroidArenaAgent/AndroidArena/HEAD/android_env2/observation.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:24:53Z", "role": "primary", "sha256": "d25f2edf9202f59615341b62fc6046ac23a6f4cb072a6038ac562506cbdc891c", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/AndroidArenaAgent/AndroidArena/HEAD/android_env2/actions.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:10Z", "role": "primary", "sha256": "257c42e980a0b14bd74bfcc381a0ecbef2b538c0f1cc1a94d22925efe354fb8e", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/AndroidArenaAgent/AndroidArena/HEAD/run_evaluator.py" } ] }, { "id": "github:thudm/android-lab", "name": "AndroidLab", "url": "https://github.com/THUDM/Android-Lab", "environments": [ "android" ], "profile": { "actions": [ "Android operations through the agent framework" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/README.md", "population": "benchmark task collection", "qualifier": "exact", "unit": "tasks", "value": 138 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/README.md", "population": "benchmark application set", "qualifier": "exact", "unit": "applications", "value": 9 } ], "evaluation": "The evaluation pipeline produces success, subgoal success, reversed redundancy, and reasonable-operation metrics; result generation requires a configured judge model.", "evidence_urls": [ "https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/README.md" ], "focus": [ "Android task completion", "subgoal completion", "operation efficiency" ], "notes": "The benchmark uses predefined virtual-device images and apps chosen to work without network/login requirements. XML and SoM modes should be reported separately. These counts concern evaluation tasks, not the separately released Android Instruct training set.", "observations": [ "UI XML", "set-of-marks views" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:41:58Z", "url": "https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "340680a9a7833d103234b8367195deebd29bfec7e25ebc50c2dc04fc3627e55b", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "340680a9a7833d103234b8367195deebd29bfec7e25ebc50c2dc04fc3627e55b", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/README.md" } ] }, { "id": "github:google-research/android_world", "name": "AndroidWorld", "url": "https://github.com/google-research/android_world", "environments": [ "android" ], "profile": { "actions": [ "Android device interactions" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/google-research/android_world/HEAD/README.md", "population": "parameterized AndroidWorld tasks", "qualifier": "exact", "unit": "task definitions", "value": 116 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/google-research/android_world/HEAD/README.md", "population": "documented AndroidWorld app set", "qualifier": "exact", "unit": "applications", "value": 20 } ], "evaluation": "Programmatic task-specific initialization, system-state success checks, and teardown.", "evidence_urls": [ "https://raw.githubusercontent.com/google-research/android_world/HEAD/README.md", "https://export.arxiv.org/api/query?id_list=2403.03186%2C2403.03788%2C2403.07718%2C2403.10170%2C2403.11905%2C2403.16971%2C2403.17918%2C2403.20271%2C2404.01549%2C2404.01744%2C2404.02475%2C2404.03648%2C2404.05427%2C2404.05719%2C2404.05955%2C2404.06474%2C2404.07972%2C2404.08755%2C2404.08860%2C2404.08948%2C2404.09992%2C2404.10887%2C2404.11459%2C2404.16006%2C2404.16048%2C2404.16054%2C2404.16660%2C2404.18074%2C2404.19296%2C2405.00516%2C2405.11120%2C2405.12842%2C2405.14573%2C2405.14751%2C2405.15793%2C2405.16247%2C2405.20309%2C2406.01014%2C2406.01623%2C2406.03679%2C2406.06947%2C2406.07822%2C2406.08184%2C2406.08340%2C2406.08451%2C2406.10227%2C2406.10819%2C2406.11317%2C2406.11896%2C2406.12373&max_results=50" ], "focus": [ "mobile task completion" ], "notes": "Random parameters produce many instances from the same task definitions. Preserve seeds and parameterization; 116 is not a count of all possible task instances.", "observations": [ "screenshots", "accessibility information" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/google-research/android_world/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "93201de5e21f2b12283a75b1e2c357781b2e6cc27eab20cc7dd7aa0f5919d839", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.task_count", "details.task_unit", "details.metrics", "details.evaluation_type", "details.benchmark_protocol", "details.leaderboard_url" ], "url": "https://raw.githubusercontent.com/google-research/android_world/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:31:28Z", "role": "primary", "sha256": "be244bfae7f89ef9449b69d85aac44295eb7b5f7e5e1bbedd76df312696676a7", "supports": [ "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?id_list=2403.03186%2C2403.03788%2C2403.07718%2C2403.10170%2C2403.11905%2C2403.16971%2C2403.17918%2C2403.20271%2C2404.01549%2C2404.01744%2C2404.02475%2C2404.03648%2C2404.05427%2C2404.05719%2C2404.05955%2C2404.06474%2C2404.07972%2C2404.08755%2C2404.08860%2C2404.08948%2C2404.09992%2C2404.10887%2C2404.11459%2C2404.16006%2C2404.16048%2C2404.16054%2C2404.16660%2C2404.18074%2C2404.19296%2C2405.00516%2C2405.11120%2C2405.12842%2C2405.14573%2C2405.14751%2C2405.15793%2C2405.16247%2C2405.20309%2C2406.01014%2C2406.01623%2C2406.03679%2C2406.06947%2C2406.07822%2C2406.08184%2C2406.08340%2C2406.08451%2C2406.10227%2C2406.10819%2C2406.11317%2C2406.11896%2C2406.12373&max_results=50" } ] }, { "id": "github:acrab-agentic-labs/appsim", "name": "AppSim-Bench", "url": "https://github.com/Acrab-Agentic-Labs/AppSim", "environments": [ "android" ], "profile": { "actions": [ "mobile GUI operations" ], "counts": [ { "basis": "reported", "evidence_url": "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=100&max_results=100&sortBy=submittedDate&sortOrder=descending", "population": "AppSim-Bench paper-described release", "qualifier": "exact", "reference_url": "https://arxiv.org/abs/2609.07712", "unit": "tasks", "value": 557 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/Acrab-Agentic-Labs/AppSim/HEAD/README.md", "population": "published simulated app set", "qualifier": "exact", "unit": "applications", "value": 17 } ], "evaluation": "Controlled backend data and outcome-based task verification.", "evidence_urls": [ "https://raw.githubusercontent.com/Acrab-Agentic-Labs/AppSim/HEAD/README.md", "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=100&max_results=100&sortBy=submittedDate&sortOrder=descending" ], "focus": [ "mobile task completion" ], "notes": "Use the simulated apps and controlled backend state supplied by the benchmark. Desktop host instructions are not the platform being evaluated.", "observations": [ "mobile interfaces" ], "setting": "interactive" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T10:42:12Z", "role": "primary", "sha256": "12dc2afe339c1ed66e4a6a4ccf5ae13d27ce32f452960341529a4b9016f69059", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/Acrab-Agentic-Labs/AppSim/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:36:32Z", "role": "primary", "sha256": "13d3fb5811d203f2e6b80eb45d30ebc0b3cfefd88e2034ffa3b1b80687bef285", "supports": [ "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=100&max_results=100&sortBy=submittedDate&sortOrder=descending" } ] }, { "id": "github:stonybrooknlp/appworld", "name": "AppWorld", "url": "https://github.com/StonyBrookNLP/appworld", "environments": [ "unspecified" ], "profile": { "actions": [ "Python code", "application API calls" ], "counts": [ { "basis": "reported", "evidence_url": "https://export.arxiv.org/api/query?id_list=2406.12814%2C2406.13719%2C2406.14056%2C2406.14250%2C2406.14314%2C2406.18082%2C2406.19263%2C2407.00993%2C2407.01476%2C2407.01489%2C2407.01511%2C2407.01558%2C2407.03037%2C2407.03913%2C2407.04346%2C2407.05291%2C2407.07061%2C2407.08850%2C2407.09018%2C2407.09295%2C2407.10956%2C2407.13032%2C2407.13063%2C2407.15673%2C2407.15711%2C2407.16741%2C2407.17490%2C2407.18901%2C2407.19056%2C2407.20183%2C2408.00203%2C2408.02544%2C2408.06327%2C2408.07199%2C2408.11824%2C2408.15978%2C2409.02453%2C2409.04566%2C2409.07429%2C2409.08264%2C2409.09354%2C2409.10741%2C2409.11295%2C2409.13711%2C2409.14337%2C2409.14818%2C2409.15441%2C2409.15637%2C2409.17140%2C2409.20566&max_results=50", "population": "original AppWorld benchmark (2024 paper)", "qualifier": "exact", "reference_url": "https://arxiv.org/abs/2407.18901", "unit": "tasks", "value": 750 }, { "basis": "reported", "evidence_url": "https://export.arxiv.org/api/query?id_list=2406.12814%2C2406.13719%2C2406.14056%2C2406.14250%2C2406.14314%2C2406.18082%2C2406.19263%2C2407.00993%2C2407.01476%2C2407.01489%2C2407.01511%2C2407.01558%2C2407.03037%2C2407.03913%2C2407.04346%2C2407.05291%2C2407.07061%2C2407.08850%2C2407.09018%2C2407.09295%2C2407.10956%2C2407.13032%2C2407.13063%2C2407.15673%2C2407.15711%2C2407.16741%2C2407.17490%2C2407.18901%2C2407.19056%2C2407.20183%2C2408.00203%2C2408.02544%2C2408.06327%2C2408.07199%2C2408.11824%2C2408.15978%2C2409.02453%2C2409.04566%2C2409.07429%2C2409.08264%2C2409.09354%2C2409.10741%2C2409.11295%2C2409.13711%2C2409.14337%2C2409.14818%2C2409.15441%2C2409.15637%2C2409.17140%2C2409.20566&max_results=50", "population": "original AppWorld environment (2024 paper)", "qualifier": "exact", "reference_url": "https://arxiv.org/abs/2407.18901", "unit": "applications", "value": 9 } ], "evaluation": "State-based unit tests check task outcomes and unintended changes.", "evidence_urls": [ "https://raw.githubusercontent.com/stonybrooknlp/appworld/HEAD/README.md", "https://export.arxiv.org/api/query?id_list=2406.12814%2C2406.13719%2C2406.14056%2C2406.14250%2C2406.14314%2C2406.18082%2C2406.19263%2C2407.00993%2C2407.01476%2C2407.01489%2C2407.01511%2C2407.01558%2C2407.03037%2C2407.03913%2C2407.04346%2C2407.05291%2C2407.07061%2C2407.08850%2C2407.09018%2C2407.09295%2C2407.10956%2C2407.13032%2C2407.13063%2C2407.15673%2C2407.15711%2C2407.16741%2C2407.17490%2C2407.18901%2C2407.19056%2C2407.20183%2C2408.00203%2C2408.02544%2C2408.06327%2C2408.07199%2C2408.11824%2C2408.15978%2C2409.02453%2C2409.04566%2C2409.07429%2C2409.08264%2C2409.09354%2C2409.10741%2C2409.11295%2C2409.13711%2C2409.14337%2C2409.14818%2C2409.15441%2C2409.15637%2C2409.17140%2C2409.20566&max_results=50" ], "focus": [ "interactive coding", "cross-app tool use" ], "notes": "This is an adjacent API/coding benchmark, not a screenshot-only task suite. Preserve the named release; task counts here refer to the original paper.", "observations": [ "API documentation", "execution feedback" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T10:48:42Z", "url": "https://raw.githubusercontent.com/StonyBrookNLP/appworld/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T10:45:03Z", "role": "primary", "sha256": "f4049141b6e520217395b7ccd9f27048c8e0f90331690dcbef300d2bc4ff7bdd", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol", "details.leaderboard_url" ], "url": "https://raw.githubusercontent.com/stonybrooknlp/appworld/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:31:32Z", "role": "primary", "sha256": "b9a83994d8ac36ccdb606a5219511f8bb35ad3948ade4f9f3e28d386221aedfb", "supports": [ "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?id_list=2406.12814%2C2406.13719%2C2406.14056%2C2406.14250%2C2406.14314%2C2406.18082%2C2406.19263%2C2407.00993%2C2407.01476%2C2407.01489%2C2407.01511%2C2407.01558%2C2407.03037%2C2407.03913%2C2407.04346%2C2407.05291%2C2407.07061%2C2407.08850%2C2407.09018%2C2407.09295%2C2407.10956%2C2407.13032%2C2407.13063%2C2407.15673%2C2407.15711%2C2407.16741%2C2407.17490%2C2407.18901%2C2407.19056%2C2407.20183%2C2408.00203%2C2408.02544%2C2408.06327%2C2408.07199%2C2408.11824%2C2408.15978%2C2409.02453%2C2409.04566%2C2409.07429%2C2409.08264%2C2409.09354%2C2409.10741%2C2409.11295%2C2409.13711%2C2409.14337%2C2409.14818%2C2409.15441%2C2409.15637%2C2409.17140%2C2409.20566&max_results=50" } ] }, { "id": "github:jylee425/b-moca", "name": "B-MoCA", "url": "https://github.com/jylee425/b-moca", "environments": [ "android" ], "profile": { "actions": [ "normalized touch/lift coordinate pairs for tap or swipe" ], "counts": [ { "basis": "computed", "evidence_url": "https://api.github.com/repos/jylee425/b-moca/git/trees/release/ver.3?recursive=1", "population": "asset/tasks/**/*.textproto in retained release/ver.3 tree; not a paper test-task total", "qualifier": "exact", "unit": "task-definition files", "value": 131 }, { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/asset/environments/config/environments_train.csv", "population": "environments_train.csv, retained release/ver.3 snapshot", "qualifier": "exact", "unit": "environment configuration rows", "value": 35 }, { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/asset/environments/config/environments_test.csv", "population": "environments_test.csv, retained release/ver.3 snapshot", "qualifier": "exact", "unit": "environment configuration rows", "value": 10 } ], "evaluation": "Task success is detected using Appium; positive task reward terminates the episode. Environment variants change device/locale/wallpaper/DPI/dark mode and are loaded as snapshots. The numeric action interface is (touch_y, touch_x, lift_y, lift_x); wrapper/tanh transformations must be kept with the evaluated agent.", "evidence_urls": [ "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/readme.md", "https://api.github.com/repos/jylee425/b-moca/git/trees/release/ver.3?recursive=1", "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/bmoca/environment/coordinator.py", "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/bmoca/environment/environment.py", "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/asset/environments/config/environments_train.csv", "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/asset/environments/config/environments_test.csv" ], "focus": [ "Android control across visual/device variation", "generalization across environment snapshots" ], "notes": "Current source inventory differs in grain from published experiment subsets: do not multiply 131 definitions by 35/10 configuration rows or label the product as evaluated tasks. Setup requires Android API 29, Appium, external app versions and generated AVD snapshots. No apps/snapshots were downloaded or run. Code Apache-2.0 does not establish data or app rights.", "observations": [ "screenshot pixels", "XML view hierarchy", "goal instruction" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:58Z", "url": "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "bd1354c8008d7d79d4883c0c037b7918392f86fab61391b39b52e507b04672ce", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/readme.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:14:05Z", "role": "primary", "sha256": "06ba40a5e0d5fb52e7bf5e5df636cb8e8bbbc56a8308dc6d528012489f7650ee", "supports": [ "details.benchmark_protocol" ], "url": "https://api.github.com/repos/jylee425/b-moca/git/trees/release/ver.3?recursive=1" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:24:53Z", "role": "primary", "sha256": "34bc0c23949732e0a8091e334d59d2d41cae49a82cde850247aa0e6d739d2464", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/bmoca/environment/coordinator.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:10Z", "role": "primary", "sha256": "d866af344c0cff5255b813576f2e9e9c5c2964e6809058b39e2b5180af71d946", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/bmoca/environment/environment.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:23:14Z", "role": "primary", "sha256": "c278cc28f8a0667b7fdc6ba638704cfbe43783cd496181231c45c865087543f3", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/asset/environments/config/environments_train.csv" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:23:15Z", "role": "primary", "sha256": "4b92933fe4f4f66e71a694b579940db11bedd5ea8f153752a44e633825b06fa3", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/jylee425/b-moca/HEAD/asset/environments/config/environments_test.csv" } ] }, { "id": "github:servicenow/browsergym", "name": "BrowserGym", "url": "https://github.com/ServiceNow/BrowserGym", "environments": [ "web" ], "profile": { "actions": [ "element-ID actions", "coordinate mouse and keyboard actions", "tab/navigation/scroll operations", "Python/Playwright action mode" ], "counts": [], "evaluation": "Gymnasium reset/step interface delegates reward, completion, messages and task info to each adapter’s validate() method. There is no universal BrowserGym success metric or shared task total; retain benchmark/adapter version and action subset for comparisons.", "evidence_urls": [ "https://raw.githubusercontent.com/ServiceNow/BrowserGym/HEAD/README.md", "https://raw.githubusercontent.com/ServiceNow/BrowserGym/HEAD/browsergym/core/src/browsergym/core/env.py", "https://raw.githubusercontent.com/ServiceNow/BrowserGym/HEAD/docs/src/core/action_space.rst", "https://api.github.com/repos/ServiceNow/BrowserGym/git/trees/main?recursive=1" ], "focus": [ "shared browser-agent evaluation interface", "benchmark adapters with separate task definitions" ], "notes": "A framework with interactive browser adapters and a static WebLINX evaluation integration, so the profile setting is mixed. README-listed integrations are not added as disjoint tasks or datasets. Core requires Playwright/Chromium; individual benchmarks have their own setup, credentials, licenses and validation rules. Apache-2.0 covers repository code only. No browser tasks or third-party project code were executed.", "observations": [ "browser screenshot", "DOM snapshot", "accessibility tree", "tab/URL state", "previous action and its error", "chat messages and task goal" ], "setting": "mixed" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/ServiceNow/BrowserGym/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "7b685115ef822989fac9093518d16c46f0202736d005eed56e7f9fc36b43f3ee", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/ServiceNow/BrowserGym/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:22:05Z", "role": "primary", "sha256": "867cc6efdfead356839fef8a26b492502b8b5e3d165d1b3c17d24aaeec268d63", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/ServiceNow/BrowserGym/HEAD/browsergym/core/src/browsergym/core/env.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:11Z", "role": "primary", "sha256": "c4bbe6b7d0cb36a5d5effab1aa5568301cead97fe4015273e1b82c667da3823a", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/ServiceNow/BrowserGym/HEAD/docs/src/core/action_space.rst" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:14:08Z", "role": "primary", "sha256": "418efbe8c65ad7ebab61d3a066339876c820a9c957506eb456b8078ab352f6bf", "supports": [ "details.benchmark_protocol" ], "url": "https://api.github.com/repos/ServiceNow/BrowserGym/git/trees/main?recursive=1" } ] }, { "id": "github:madeagents/colorbench", "name": "ColorBench", "url": "https://github.com/MadeAgents/ColorBench", "environments": [ "android" ], "profile": { "actions": [ "mobile GUI operations along graph transitions" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/MadeAgents/ColorBench/HEAD/README.md", "population": "reported ColorBench benchmark", "qualifier": "exact", "unit": "tasks", "value": 175 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/MadeAgents/ColorBench/HEAD/README.md", "population": "reported ColorBench benchmark", "qualifier": "exact", "unit": "cross-app tasks", "value": 101 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/MadeAgents/ColorBench/HEAD/README.md", "population": "reported ColorBench benchmark", "qualifier": "exact", "unit": "single-app tasks", "value": 74 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/MadeAgents/ColorBench/HEAD/README.md", "population": "reported ColorBench benchmark", "qualifier": "exact", "unit": "mobile applications", "value": 21 } ], "evaluation": "The publisher reports success rate, completion rate and atomic-capability diagnostics using graph-based evaluation.", "evidence_urls": [ "https://raw.githubusercontent.com/MadeAgents/ColorBench/HEAD/README.md" ], "focus": [ "long-horizon mobile task completion", "multi-path graph interaction" ], "notes": "The interaction environment replays graph states rather than running real user accounts. The 101 and 74 subsets partition the reported 175 tasks; do not add the three counts.", "observations": [ "mobile screenshots represented as graph states" ], "setting": "interactive" }, "licenses": { "code": { "basis": "Repository source LICENSE.", "declared": "Apache-2.0", "retrieved_at": "2026-10-17T11:22:54Z", "url": "https://raw.githubusercontent.com/MadeAgents/ColorBench/HEAD/LICENSE" }, "data": { "basis": "Publisher metadata for the separately hosted screenshot archive; underlying third-party content retains applicable rights.", "declared": "apache-2.0", "retrieved_at": "2026-10-17T11:23:06Z", "url": "https://huggingface.co/datasets/Norsheep/ColorBench/raw/main/README.md" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:18:16Z", "role": "primary", "sha256": "5c0190fffb68b91c1855a408f6a5e252871d94000b95fa258504a3c100600cec", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/MadeAgents/ColorBench/HEAD/README.md" } ] }, { "id": "github:camel-ai/crab", "name": "CRAB", "url": "https://github.com/camel-ai/crab", "environments": [ "desktop", "android" ], "profile": { "actions": [ "Python-defined environment actions" ], "counts": [ { "basis": "reported", "evidence_url": "https://export.arxiv.org/api/query?id_list=2406.12814%2C2406.13719%2C2406.14056%2C2406.14250%2C2406.14314%2C2406.18082%2C2406.19263%2C2407.00993%2C2407.01476%2C2407.01489%2C2407.01511%2C2407.01558%2C2407.03037%2C2407.03913%2C2407.04346%2C2407.05291%2C2407.07061%2C2407.08850%2C2407.09018%2C2407.09295%2C2407.10956%2C2407.13032%2C2407.13063%2C2407.15673%2C2407.15711%2C2407.16741%2C2407.17490%2C2407.18901%2C2407.19056%2C2407.20183%2C2408.00203%2C2408.02544%2C2408.06327%2C2408.07199%2C2408.11824%2C2408.15978%2C2409.02453%2C2409.04566%2C2409.07429%2C2409.08264%2C2409.09354%2C2409.10741%2C2409.11295%2C2409.13711%2C2409.14337%2C2409.14818%2C2409.15441%2C2409.15637%2C2409.17140%2C2409.20566&max_results=50", "population": "CRAB Benchmark-v0 (paper)", "qualifier": "exact", "reference_url": "https://arxiv.org/abs/2407.01511", "unit": "tasks", "value": 120 } ], "evaluation": "Graph-based evaluators measure progress through task requirements.", "evidence_urls": [ "https://raw.githubusercontent.com/camel-ai/crab/HEAD/README.md", "https://export.arxiv.org/api/query?id_list=2406.12814%2C2406.13719%2C2406.14056%2C2406.14250%2C2406.14314%2C2406.18082%2C2406.19263%2C2407.00993%2C2407.01476%2C2407.01489%2C2407.01511%2C2407.01558%2C2407.03037%2C2407.03913%2C2407.04346%2C2407.05291%2C2407.07061%2C2407.08850%2C2407.09018%2C2407.09295%2C2407.10956%2C2407.13032%2C2407.13063%2C2407.15673%2C2407.15711%2C2407.16741%2C2407.17490%2C2407.18901%2C2407.19056%2C2407.20183%2C2408.00203%2C2408.02544%2C2408.06327%2C2408.07199%2C2408.11824%2C2408.15978%2C2409.02453%2C2409.04566%2C2409.07429%2C2409.08264%2C2409.09354%2C2409.10741%2C2409.11295%2C2409.13711%2C2409.14337%2C2409.14818%2C2409.15441%2C2409.15637%2C2409.17140%2C2409.20566&max_results=50" ], "focus": [ "cross-environment tasks" ], "notes": "CRAB is also an extensible framework. The 120-task count belongs to Benchmark-v0, not every environment that can be created with the framework.", "observations": [], "setting": "interactive" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "7b894c5474ba78294427cfd4ef0eea8cb22d76dfa0c2e8c7f01e31ad927064e9", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/camel-ai/crab/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:31:32Z", "role": "primary", "sha256": "b9a83994d8ac36ccdb606a5219511f8bb35ad3948ade4f9f3e28d386221aedfb", "supports": [ "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?id_list=2406.12814%2C2406.13719%2C2406.14056%2C2406.14250%2C2406.14314%2C2406.18082%2C2406.19263%2C2407.00993%2C2407.01476%2C2407.01489%2C2407.01511%2C2407.01558%2C2407.03037%2C2407.03913%2C2407.04346%2C2407.05291%2C2407.07061%2C2407.08850%2C2407.09018%2C2407.09295%2C2407.10956%2C2407.13032%2C2407.13063%2C2407.15673%2C2407.15711%2C2407.16741%2C2407.17490%2C2407.18901%2C2407.19056%2C2407.20183%2C2408.00203%2C2408.02544%2C2408.06327%2C2408.07199%2C2408.11824%2C2408.15978%2C2409.02453%2C2409.04566%2C2409.07429%2C2409.08264%2C2409.09354%2C2409.10741%2C2409.11295%2C2409.13711%2C2409.14337%2C2409.14818%2C2409.15441%2C2409.15637%2C2409.17140%2C2409.20566&max_results=50" } ] }, { "id": "url:d5140d438466c1e1348a", "name": "CUAVerifierBench", "url": "https://huggingface.co/datasets/microsoft/CUAVerifierBench", "environments": [ "web" ], "profile": { "actions": [ "success verdict", "process score" ], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/microsoft/CUAVerifierBench/raw/main/README.md", "population": "fara7b_om2w_browserbase split", "qualifier": "exact", "unit": "trajectories", "value": 106 }, { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/microsoft/CUAVerifierBench/raw/main/README.md", "population": "internal split", "qualifier": "exact", "unit": "trajectories", "value": 154 }, { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/microsoft/CUAVerifierBench/raw/main/README.md", "population": "fara7b_om2w_browserbase split", "qualifier": "exact", "unit": "annotation rows", "value": 215 }, { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/microsoft/CUAVerifierBench/raw/main/README.md", "population": "internal split", "qualifier": "exact", "unit": "annotation rows", "value": 154 } ], "evaluation": "Agreement metrics including Cohen’s kappa, accuracy and F1 against human annotations; review stages must be kept distinct.", "evidence_urls": [ "https://huggingface.co/datasets/microsoft/CUAVerifierBench/raw/main/README.md" ], "focus": [ "verifier-human agreement", "process quality" ], "notes": "Trajectory rows and reviewer rows have different grains. Internal task labels do not make the released Parquet files private. No payload data was downloaded.", "observations": [ "screenshots", "action logs", "final answers" ], "setting": "offline" }, "licenses": { "code": null, "data": { "basis": "publisher model/dataset card metadata; third-party assets retain their own terms", "declared": "mit", "retrieved_at": "2026-10-17T02:42:32Z", "url": "https://huggingface.co/datasets/microsoft/CUAVerifierBench/raw/main/README.md" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:42:32Z", "role": "primary", "sha256": "44d7190b72ae489297f59d6fadff635ced6dd067c845c77b04f187a7b2d7bec7", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details", "licenses.data" ], "url": "https://huggingface.co/datasets/microsoft/CUAVerifierBench/raw/main/README.md" } ] }, { "id": "github:wwh0411/fedmabench", "name": "FedMABench", "url": "https://github.com/wwh0411/FedMABench", "environments": [ "android" ], "profile": { "actions": [ "model predictions on mobile-agent examples" ], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/wwh0411/FedMABench/raw/main/README.md", "population": "released FedMABench partition families", "qualifier": "exact", "unit": "dataset families", "value": 6 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/wwh0411/FedMABench/HEAD/README.md", "population": "publisher benchmark overview", "qualifier": "exact", "unit": "heterogeneity types", "value": 3 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/wwh0411/FedMABench/HEAD/README.md", "population": "publisher benchmark overview", "qualifier": "exact", "unit": "federated learning algorithms", "value": 8 } ], "evaluation": "The provided evaluation workflow performs model inference over selected partitions and invokes test_swift_fed.py for accuracy scoring.", "evidence_urls": [ "https://raw.githubusercontent.com/wwh0411/FedMABench/HEAD/README.md", "https://huggingface.co/datasets/wwh0411/FedMABench/raw/main/README.md" ], "focus": [ "mobile-agent federated training", "data heterogeneity" ], "notes": "The dataset-family count does not count tasks, apps, screenshots or client instances. Code and data licenses are unknown; external image sources are required.", "observations": [ "mobile-agent examples with externally sourced screenshots" ], "setting": "offline" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:18:17Z", "role": "primary", "sha256": "b49e2b419918078ef9c68f3c9becc6c13d8fe530c165a36bdd61e370746eeea8", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/wwh0411/FedMABench/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:23:06Z", "role": "primary", "sha256": "2874269995d8c9f5dabe266433ecf724bf54c27af599d899f3f0090b8ee37cdc", "supports": [ "summary", "details.benchmark_protocol", "licenses.data" ], "url": "https://huggingface.co/datasets/wwh0411/FedMABench/raw/main/README.md" } ] }, { "id": "github:ant-research/guihallubench", "name": "GUIHalluBench", "url": "https://github.com/ant-research/GUIHalluBench", "environments": [ "cross-platform" ], "profile": { "actions": [ "list of UI element boxes [x1,y1,x2,y2] and text descriptions" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ant-research/GUIHalluBench/HEAD/README.md", "population": "English annotation subset reported by README", "qualifier": "exact", "unit": "page-annotation entries", "value": 1000 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ant-research/GUIHalluBench/HEAD/README.md", "population": "Chinese annotation subset reported by README", "qualifier": "exact", "unit": "page-annotation entries", "value": 1000 } ], "evaluation": "Metrics use greedy one-to-one matching in descending IoU order, then micro precision/recall/F1 and lowercased difflib.SequenceMatcher description similarity on matched pairs. Actual metrics.py accepts IoU >= 0.5 at its default threshold; README prose says > 0.5. Predictions align to annotations by image_path; callers must supply complete prediction coverage because missing predictions are not independently enumerated by evaluate_dataset.", "evidence_urls": [ "https://raw.githubusercontent.com/ant-research/GUIHalluBench/HEAD/README.md", "https://raw.githubusercontent.com/ant-research/GUIHalluBench/HEAD/src/metrics.py", "https://api.github.com/repos/ant-research/GUIHalluBench/git/trees/master?recursive=1", "https://raw.githubusercontent.com/ant-research/GUIHalluBench/HEAD/LEGAL.md", "https://raw.githubusercontent.com/ant-research/GUIHalluBench/HEAD/.gitattributes" ], "focus": [ "GUI page-element detection", "element-description consistency" ], "notes": "Each annotation entry represents a page, not one UI element. README’s illustrative num_samples=4796 output is not adopted as a corpus count. Public tree contains both annotation JSON files and an images.tar.gz Git-LFS pointer; payload availability/integrity was not tested by downloading images. LEGAL.md only governs comment-language precedence, not a code/data license grant; both slots remain unknown. No benchmark run was performed.", "observations": [ "page screenshot" ], "setting": "offline" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:59Z", "role": "primary", "sha256": "d9c476d6960a4ef99e20d4c3b5d35a63f3ae5e61fc20e9dea00b11e64b74c9e9", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/ant-research/GUIHalluBench/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:09Z", "role": "primary", "sha256": "f36707dfcd3e83fc9d7f57b50d0c27007801e5029cc1ac255e000560ecf778fc", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/ant-research/GUIHalluBench/HEAD/src/metrics.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:14:07Z", "role": "primary", "sha256": "c0896bc90d515b7bd8fe86b062aeb199bd47570e6eca8474986f5b1ef1d98dbd", "supports": [ "details.benchmark_protocol" ], "url": "https://api.github.com/repos/ant-research/GUIHalluBench/git/trees/master?recursive=1" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:09Z", "role": "primary", "sha256": "adba50d9794b9ef3f7ec8cbc680f7f1fa3fbf9df0ac8d1f9b9ccab6d941bc11b", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/ant-research/GUIHalluBench/HEAD/LEGAL.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:34:56Z", "role": "primary", "sha256": "8f35fa75be42a7a53e4f96de1b62be84962d2cf1beb8b23dfcfaecfaea4a6b90", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/ant-research/GUIHalluBench/HEAD/.gitattributes" } ] }, { "id": "github:ljang0/iosworld", "name": "iOSWorld", "url": "https://github.com/ljang0/iOSWorld", "environments": [ "ios" ], "profile": { "actions": [ "touch actions", "MCP tools (optional)" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ljang0/iOSWorld/HEAD/README.md", "population": "tasks.json benchmark", "qualifier": "exact", "unit": "tasks", "value": 133 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ljang0/iOSWorld/HEAD/README.md", "population": "bundled SwiftUI apps", "qualifier": "exact", "unit": "applications", "value": 26 } ], "evaluation": "Rubric satisfaction per task; full-task pass requires every criterion to be satisfied.", "evidence_urls": [ "https://raw.githubusercontent.com/ljang0/iOSWorld/HEAD/README.md" ], "focus": [ "personalized phone tasks", "cross-app workflows" ], "notes": "Vision-only, vision-plus-XML, and MCP modes expose different information and tools. The documented default evaluator is model-based; compare the same mode and judge.", "observations": [ "screenshots", "accessibility XML (optional)" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T10:48:41Z", "url": "https://raw.githubusercontent.com/ljang0/iOSWorld/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T10:42:12Z", "role": "primary", "sha256": "706f9d770b65a1238e56f74f7dd5089c7756cc88e0ac09d80c7b2b3085a09b27", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.task_count", "details.metrics", "details.evaluation_type", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/ljang0/iOSWorld/HEAD/README.md" } ] }, { "id": "github:llamatouch/llamatouch", "name": "LlamaTouch", "url": "https://github.com/LlamaTouch/LlamaTouch", "environments": [ "android" ], "profile": { "actions": [ "device actions through AgentEnv" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/LlamaTouch/LlamaTouch/HEAD/README.md", "population": "complete task collection", "qualifier": "exact", "unit": "tasks", "value": 495 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/LlamaTouch/LlamaTouch/HEAD/README.md", "population": "AITW-derived subset", "qualifier": "exact", "unit": "tasks", "value": 102 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/LlamaTouch/LlamaTouch/HEAD/README.md", "population": "self-constructed subset", "qualifier": "exact", "unit": "tasks", "value": 393 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/LlamaTouch/LlamaTouch/HEAD/README.md", "population": "self-constructed subset", "qualifier": "exact", "unit": "Android applications", "value": 46 } ], "evaluation": "Checks whether the recorded execution visits all annotated essential states rather than requiring an identical action sequence.", "evidence_urls": [ "https://raw.githubusercontent.com/LlamaTouch/LlamaTouch/HEAD/README.md" ], "focus": [ "mobile task completion", "essential-state evaluation" ], "notes": "Application coverage refers to the self-constructed subset. Task execution and essential-state annotation are separate components; counts are publisher-reported, not a fresh task-file recount.", "observations": [ "screenshots", "view hierarchies", "Android activity names" ], "setting": "interactive" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "4c4fe0fa92961f7286b6553b13117b2942e8482a48b7120a79e57dce2f8d42b9", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/LlamaTouch/LlamaTouch/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "4c4fe0fa92961f7286b6553b13117b2942e8482a48b7120a79e57dce2f8d42b9", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/LlamaTouch/LlamaTouch/HEAD/README.md" } ] }, { "id": "github:jetastra/macagentbench", "name": "MacAgentBench", "url": "https://github.com/JetAstra/MacAgentBench", "environments": [ "desktop", "macos" ], "profile": { "actions": [ "GUI interactions", "command-line operations" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/README.md", "population": "documented MacAgentBench suite", "qualifier": "exact", "unit": "tasks", "value": 676 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/README.md", "population": "documented application coverage", "qualifier": "exact", "unit": "applications", "value": 25 } ], "evaluation": "Deterministic rule-based evaluators with intermediate checkpoints.", "evidence_urls": [ "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/README.md", "https://export.arxiv.org/api/query?id_list=2605.06761%2C2605.07110%2C2605.07134%2C2605.09497%2C2605.11212%2C2605.12481%2C2605.12501%2C2605.12549%2C2605.13631%2C2605.14290%2C2605.14747%2C2605.15030%2C2605.15542%2C2605.15777%2C2605.16024%2C2605.16116%2C2605.16402%2C2605.17320%2C2605.18048%2C2605.18652%2C2605.18758%2C2605.19260%2C2605.19769%2C2605.21470%2C2605.25160%2C2605.25624%2C2605.25707%2C2605.26114%2C2605.27761%2C2605.28116%2C2605.28534%2C2605.28646%2C2605.28775%2C2605.29119%2C2605.29324%2C2605.29447%2C2605.29486%2C2605.29534%2C2605.29615%2C2606.06322%2C2606.09426%2C2606.11042%2C2606.22557%2C2606.23189%2C2606.24551%2C2608.17659%2C2609.22000%2C2609.34139&max_results=48" ], "focus": [ "macOS task completion", "cross-application workflows" ], "notes": "The benchmark evaluates multiple agent frameworks and GUI/CLI workflows. Framework skills and interfaces affect the task setting; the inspected README does not define one universal observation representation.", "observations": [], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:41:59Z", "url": "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:59Z", "role": "primary", "sha256": "f1f8c0cd5d984e4026abea858b54f0f1427c018cad7e75322e744f1e070225a9", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol", "details.leaderboard_url" ], "url": "https://raw.githubusercontent.com/JetAstra/MacAgentBench/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:32:03Z", "role": "primary", "sha256": "2adad07a9ec7189aea25e48c76fa28206eda97f8974d5745cadda9529103bb1a", "supports": [ "details.benchmark_protocol", "relations" ], "url": "https://export.arxiv.org/api/query?id_list=2605.06761%2C2605.07110%2C2605.07134%2C2605.09497%2C2605.11212%2C2605.12481%2C2605.12501%2C2605.12549%2C2605.13631%2C2605.14290%2C2605.14747%2C2605.15030%2C2605.15542%2C2605.15777%2C2605.16024%2C2605.16116%2C2605.16402%2C2605.17320%2C2605.18048%2C2605.18652%2C2605.18758%2C2605.19260%2C2605.19769%2C2605.21470%2C2605.25160%2C2605.25624%2C2605.25707%2C2605.26114%2C2605.27761%2C2605.28116%2C2605.28534%2C2605.28646%2C2605.28775%2C2605.29119%2C2605.29324%2C2605.29447%2C2605.29486%2C2605.29534%2C2605.29615%2C2606.06322%2C2606.09426%2C2606.11042%2C2606.22557%2C2606.23189%2C2606.24551%2C2608.17659%2C2609.22000%2C2609.34139&max_results=48" } ] }, { "id": "github:osu-nlp-group/mind2web", "name": "Mind2Web", "url": "https://github.com/OSU-NLP-Group/Mind2Web", "environments": [ "web" ], "profile": { "actions": [ "CLICK", "TYPE", "SELECT" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/OSU-NLP-Group/Mind2Web/HEAD/README.md", "population": "train split", "qualifier": "exact", "unit": "task instances", "value": 1009 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/OSU-NLP-Group/Mind2Web/HEAD/README.md", "population": "cross-task test split", "qualifier": "exact", "unit": "task instances", "value": 252 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/OSU-NLP-Group/Mind2Web/HEAD/README.md", "population": "cross-website test split", "qualifier": "exact", "unit": "task instances", "value": 177 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/OSU-NLP-Group/Mind2Web/HEAD/README.md", "population": "cross-domain test split", "qualifier": "exact", "unit": "task instances", "value": 912 } ], "evaluation": "Offline candidate and action evaluation against recorded demonstrations; the repository distinguishes macro from micro averaging.", "evidence_urls": [ "https://raw.githubusercontent.com/OSU-NLP-Group/Mind2Web/HEAD/README.md" ], "focus": [ "web action prediction", "generalization to unseen sites and domains" ], "notes": "These are reported split inventories, not online task-success measurements. Original Mind2Web, Multimodal-Mind2Web, and Online-Mind2Web have different inputs/protocols. The authors require macro-average metrics for comparison with the paper; test archives remain upstream.", "observations": [ "task instructions", "HTML", "recorded action history" ], "setting": "offline" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/OSU-NLP-Group/Mind2Web/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "157dbd840264ea29f958c0b7383d63783845c573b3e19a1f39b773be22e0b32d", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/OSU-NLP-Group/Mind2Web/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "157dbd840264ea29f958c0b7383d63783845c573b3e19a1f39b773be22e0b32d", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/OSU-NLP-Group/Mind2Web/HEAD/README.md" } ] }, { "id": "github:farama-foundation/miniwob-plusplus", "name": "MiniWoB++", "url": "https://github.com/Farama-Foundation/miniwob-plusplus", "environments": [ "web" ], "profile": { "actions": [ "configurable coordinate actions", "element-reference actions", "keyboard input" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/Farama-Foundation/miniwob-plusplus/HEAD/README.md", "population": "README collection size", "qualifier": "more-than", "unit": "web environments", "value": 100 } ], "evaluation": "", "evidence_urls": [ "https://raw.githubusercontent.com/Farama-Foundation/miniwob-plusplus/HEAD/README.md", "https://miniwob.farama.org/content/observation_space/", "https://miniwob.farama.org/content/action_space/" ], "focus": [ "small web interaction tasks" ], "notes": "The README gives a lower bound, not an exact active-environment inventory. Action presets expose different capabilities, so comparisons must identify the preset. Environment-specific reward details are not normalized here. The repository is in maintenance mode.", "observations": [ "task instructions", "screenshots", "visible DOM-element attributes" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:42:07Z", "url": "https://raw.githubusercontent.com/Farama-Foundation/miniwob-plusplus/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:40:54Z", "role": "primary", "sha256": "90499776fa1b23e08ac85190ad3be709caa9172ff4f2a0a106b0fa350568e86f", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/Farama-Foundation/miniwob-plusplus/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:40:54Z", "role": "primary", "sha256": "90499776fa1b23e08ac85190ad3be709caa9172ff4f2a0a106b0fa350568e86f", "supports": [ "details.benchmark_protocol", "relations", "notes" ], "url": "https://raw.githubusercontent.com/Farama-Foundation/miniwob-plusplus/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:03:23Z", "role": "primary", "sha256": "511039e0e3c26b117e7e0a001746c26ec6d94a8ab8c467f0be0bc1a0b01e870a", "supports": [ "details.benchmark_protocol" ], "url": "https://miniwob.farama.org/content/observation_space/" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:03:23Z", "role": "primary", "sha256": "ae7bf6a3b98b198b985d7ca7b41cb2bdab3cf48e6445467ad9a8d4a2c8df240f", "supports": [ "details.benchmark_protocol" ], "url": "https://miniwob.farama.org/content/action_space/" } ] }, { "id": "github:open-compass/mmbench-gui", "name": "MMBench-GUI", "url": "https://github.com/open-compass/MMBench-GUI", "environments": [ "cross-platform" ], "profile": { "actions": [], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/open-compass/MMBench-GUI/HEAD/README.md", "population": "reported multi-level benchmark", "qualifier": "more-than", "unit": "tasks", "value": 8000 } ], "evaluation": "Level-specific evaluation; Efficiency–Quality Area (EQA) combines navigation quality and efficiency.", "evidence_urls": [ "https://raw.githubusercontent.com/open-compass/MMBench-GUI/HEAD/README.md" ], "focus": [ "screen understanding", "grounding", "task automation", "task collaboration" ], "notes": "The README documents four levels, but its release notes and TODOs distinguish released L1/L2 evaluation from pending L3/L4 integration. Do not treat every level as the same evaluator or a complete implemented interface.", "observations": [ "screenshots" ], "setting": "mixed" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "35db9fca3ed1c1a3513924fefff4ccd224bc70cd7eb4bf8c54fcb9048851988e", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/open-compass/MMBench-GUI/HEAD/README.md" } ] }, { "id": "url:b53e03a6e4ca4226d126", "name": "MobBench task definitions", "url": "https://huggingface.co/datasets/OpenDFM/MobA-MobBench", "environments": [ "android" ], "profile": { "actions": [ "mobile GUI actions through a separately configured agent" ], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/OpenDFM/MobA-MobBench/raw/main/README.md", "population": "MobBench tasks used in the paper", "qualifier": "exact", "unit": "tasks", "value": 50 } ], "evaluation": "The publisher reports a fifty-task real-device evaluation; the inspected release does not establish an automatic correctness judge.", "evidence_urls": [ "https://huggingface.co/datasets/OpenDFM/MobA-MobBench/raw/main/README.md" ], "focus": [ "mobile task completion" ], "notes": "Task definitions are distinct from device images, agent execution and automated scoring.", "observations": [ "Android device interface" ], "setting": "interactive" }, "licenses": { "code": null, "data": { "basis": "Dataset-card declaration for the MobBench task-definition release only.", "declared": "cc-by-nc-sa-4.0", "retrieved_at": "2026-10-17T12:32:07Z", "url": "https://huggingface.co/datasets/OpenDFM/MobA-MobBench/raw/main/README.md" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T12:32:07Z", "role": "primary", "sha256": "231a8f483256c774840c99383ab6536520a0aed0f04d62b4d64f1f59477b906e", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details.benchmark_protocol", "licenses.data" ], "url": "https://huggingface.co/datasets/OpenDFM/MobA-MobBench/raw/main/README.md" } ] }, { "id": "url:1d5b85321b98765141fc", "name": "Mobile-Bench-v2 released data", "url": "https://huggingface.co/datasets/xwk123/MobileBench-v2", "environments": [ "android" ], "profile": { "actions": [ "mobile UI actions", "clarification responses" ], "counts": [], "evaluation": "Paper defines multi-path and noise/ambiguity evaluations; the public card identifies which files are full common-task sets and which are samples.", "evidence_urls": [ "https://huggingface.co/datasets/xwk123/MobileBench-v2/raw/main/README.md", "https://export.arxiv.org/api/query?id_list=2503.16788%2C2503.17709%2C2503.18492%2C2503.19537%2C2503.20978%2C2503.21620%2C2503.23434%2C2504.01382%2C2504.07981%2C2504.10445%2C2504.10458%2C2504.11257%2C2504.11281%2C2504.12679%2C2504.13805%2C2504.13865%2C2504.14239%2C2504.18575%2C2504.19838%2C2504.20464%2C2505.10924%2C2505.11891%2C2505.12299%2C2505.12632%2C2505.13227%2C2505.13887%2C2505.13909%2C2505.14289%2C2505.15810%2C2505.16282%2C2505.16827%2C2505.18829%2C2505.19897%2C2505.21496%2C2505.21499%2C2505.21936%2C2505.22648%2C2505.23762%2C2505.24838%2C2506.00618%2C2506.01391%2C2506.01968%2C2506.02456%2C2506.03143%2C2506.03569%2C2506.04135%2C2506.04614%2C2506.07672%2C2506.08012%2C2506.08972&max_results=50" ], "focus": [ "offline multi-path evaluation", "noisy interfaces", "ambiguous instructions" ], "notes": "Release completeness differs by split. No full-benchmark sample total is inferred from the archive manifest.", "observations": [ "mobile screenshots", "XML UI state", "instructions" ], "setting": "offline" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T04:01:33Z", "role": "primary", "sha256": "13424be795c7b75b8c2195469c1252a7b3ec633e8bb31f1f6d7e44476611cbe9", "supports": [ "name", "kind", "summary", "topics", "scope", "relations", "details", "licenses.data" ], "url": "https://huggingface.co/datasets/xwk123/MobileBench-v2/raw/main/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:31:42Z", "role": "primary", "sha256": "afe4e75ffb8a5c44f4c088a90ffa5f53553036f514aa768f5aee0c5aa38d731e", "supports": [ "relations", "summary" ], "url": "https://export.arxiv.org/api/query?id_list=2503.16788%2C2503.17709%2C2503.18492%2C2503.19537%2C2503.20978%2C2503.21620%2C2503.23434%2C2504.01382%2C2504.07981%2C2504.10445%2C2504.10458%2C2504.11257%2C2504.11281%2C2504.12679%2C2504.13805%2C2504.13865%2C2504.14239%2C2504.18575%2C2504.19838%2C2504.20464%2C2505.10924%2C2505.11891%2C2505.12299%2C2505.12632%2C2505.13227%2C2505.13887%2C2505.13909%2C2505.14289%2C2505.15810%2C2505.16282%2C2505.16827%2C2505.18829%2C2505.19897%2C2505.21496%2C2505.21499%2C2505.21936%2C2505.22648%2C2505.23762%2C2505.24838%2C2506.00618%2C2506.01391%2C2506.01968%2C2506.02456%2C2506.03143%2C2506.03569%2C2506.04135%2C2506.04614%2C2506.07672%2C2506.08012%2C2506.08972&max_results=50" } ] }, { "id": "url:342b081c3668a8bbe155", "name": "Mobile-Eval-E task definitions", "url": "https://huggingface.co/datasets/mikewang/mobile_eval_e", "environments": [ "android" ], "profile": { "actions": [], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/mikewang/mobile_eval_e/raw/b8eca82988ed8d4f9dec01eaa9d51abf4b35aa83/README.md", "population": "Hub test split at revision b8eca829", "qualifier": "exact", "unit": "task-definition records", "value": 25 } ], "evaluation": "The published task schema contains rubrics and human_reference_operations. No automated scoring protocol or single numeric agent metric is established by the inspected card; execution uses separately configured Android tooling.", "evidence_urls": [ "https://huggingface.co/datasets/mikewang/mobile_eval_e/raw/b8eca82988ed8d4f9dec01eaa9d51abf4b35aa83/README.md", "https://huggingface.co/api/datasets/mikewang/mobile_eval_e", "https://raw.githubusercontent.com/X-PLUG/MobileAgent/HEAD/Mobile-Agent-E/README.md", "https://export.arxiv.org/api/query?id_list=2412.04454%2C2412.04626%2C2412.05467%2C2412.09362%2C2412.09605%2C2412.10342%2C2412.10742%2C2412.10840%2C2412.13103%2C2412.13194%2C2412.13501%2C2412.14161%2C2412.14470%2C2412.16256%2C2412.17589%2C2412.18116%2C2412.18426%2C2412.19723%2C2501.01149%2C2501.04575%2C2501.07572%2C2501.11733%2C2501.12326%2C2501.13896%2C2501.16150%2C2502.01600%2C2502.01977%2C2502.02982%2C2502.06395%2C2502.07949%2C2502.08047%2C2502.08226%2C2502.11357%2C2502.13053%2C2502.13923%2C2502.14282%2C2502.18906%2C2503.00401%2C2503.02268%2C2503.03196%2C2503.04730%2C2503.05143%2C2503.06470%2C2503.09263%2C2503.11069%2C2503.11170%2C2503.12687%2C2503.15661%2C2503.15937%2C2503.16465&max_results=50" ], "focus": [ "long-horizon multi-app Android task execution" ], "notes": "Features are task_id, instruction, type, apps, scenario, rubrics and human-reference operation strings. A task record is not an action count or recorded screenshot trajectory. Public Parquet inventory was checked but no sample rows were downloaded; companion README limits the tested agent platform to Android.", "observations": [ "task instructions", "app/scenario metadata" ], "setting": "interactive" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:27:15Z", "role": "primary", "sha256": "f8515fbce73ea0ef3d1d558d4567bb949244e6384e61c956ba3ddbc27294f0ae", "supports": [ "name", "kind", "environments", "scope", "notes", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/X-PLUG/MobileAgent/HEAD/Mobile-Agent-E/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:33:29Z", "role": "primary", "sha256": "f9b10eada9e0a64e26f1a7d477768f218b807103199855d893b3ab07feddc755", "supports": [ "identity", "release_status", "details.hub_id", "details.hub_revision", "details.gated", "details.disabled", "details.download_url", "details.benchmark_protocol" ], "url": "https://huggingface.co/api/datasets/mikewang/mobile_eval_e" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:35:15Z", "role": "primary", "sha256": "9fdf43753fbfac9868d8bec57fa5601db3e03ed5c30fb981f9da88f6b75ffcc5", "supports": [ "summary", "details.benchmark_protocol", "licenses.data" ], "url": "https://huggingface.co/datasets/mikewang/mobile_eval_e/raw/b8eca82988ed8d4f9dec01eaa9d51abf4b35aa83/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:31:38Z", "role": "primary", "sha256": "e96b86015c763c780e0430a8d0867b96cae947f50e888b3441c85abcaa42de78", "supports": [ "summary", "topics", "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?id_list=2412.04454%2C2412.04626%2C2412.05467%2C2412.09362%2C2412.09605%2C2412.10342%2C2412.10742%2C2412.10840%2C2412.13103%2C2412.13194%2C2412.13501%2C2412.14161%2C2412.14470%2C2412.16256%2C2412.17589%2C2412.18116%2C2412.18426%2C2412.19723%2C2501.01149%2C2501.04575%2C2501.07572%2C2501.11733%2C2501.12326%2C2501.13896%2C2501.16150%2C2502.01600%2C2502.01977%2C2502.02982%2C2502.06395%2C2502.07949%2C2502.08047%2C2502.08226%2C2502.11357%2C2502.13053%2C2502.13923%2C2502.14282%2C2502.18906%2C2503.00401%2C2503.02268%2C2503.03196%2C2503.04730%2C2503.05143%2C2503.06470%2C2503.09263%2C2503.11069%2C2503.11170%2C2503.12687%2C2503.15661%2C2503.15937%2C2503.16465&max_results=50" } ] }, { "id": "url:d7a4f34a6a5202af5f64", "name": "MobileGUI-VBench", "url": "https://huggingface.co/datasets/vivo-ai/MobileGUI-VBench", "environments": [ "unspecified" ], "profile": { "actions": [ "mobile interface operations", "clarification requests" ], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/vivo-ai/MobileGUI-VBench/raw/main/README.md", "population": "MobileGUI-VBench release", "qualifier": "exact", "unit": "tasks", "value": 150 }, { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/vivo-ai/MobileGUI-VBench/raw/main/README.md", "population": "MobileGUI-VBench release", "qualifier": "exact", "unit": "mobile apps", "value": 40 } ], "evaluation": "Published task definitions include preconditions and clarification requirements; the metadata review does not independently validate the reported agent scores.", "evidence_urls": [ "https://huggingface.co/datasets/vivo-ai/MobileGUI-VBench/raw/main/README.md" ], "focus": [ "mobile task completion", "clarification behavior", "cross-app tasks" ], "notes": "The card describes 140 single-app and 10 cross-app tasks. Task and app counts are different units.", "observations": [ "mobile app interfaces" ], "setting": "interactive" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:42:32Z", "role": "primary", "sha256": "422ff6a557a1aabd8f6c6ef26e5fd05b992f4acf0e5128ab8bc437021fcc5ea7", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://huggingface.co/datasets/vivo-ai/MobileGUI-VBench/raw/main/README.md" } ] }, { "id": "github:purewhiter/mobilegym", "name": "MobileGym", "url": "https://github.com/Purewhiter/mobilegym", "environments": [ "web" ], "profile": { "actions": [ "tap", "type", "swipe", "back", "home", "drag", "complete" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/README.md", "population": "publisher benchmark overview", "qualifier": "exact", "unit": "task templates", "value": 416 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/README.md", "population": "publisher benchmark overview", "qualifier": "exact", "unit": "simulated apps", "value": 28 }, { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/bench_env/splits/train.txt", "population": "bench_env/splits/train.txt", "qualifier": "exact", "unit": "task identifiers", "value": 160 }, { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/bench_env/splits/test.txt", "population": "bench_env/splits/test.txt", "qualifier": "exact", "unit": "task identifiers", "value": 256 } ], "evaluation": "Deterministic task judges inspect state changes; reported metrics distinguish success, progress, false completion and unexpected side effects.", "evidence_urls": [ "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/README.md", "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/bench_env/splits/train.txt", "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/bench_env/splits/test.txt" ], "focus": [ "mobile GUI task completion in simulation", "state verification and parallel training" ], "notes": "The split-ID inventory is not a count of sampled trajectories, live Android apps or real user accounts. Extra runtime/data dependencies and third-party content terms remain distinct.", "observations": [ "simulated mobile screenshots", "structured simulator state for judges" ], "setting": "interactive" }, "licenses": { "code": { "basis": "Core source LICENSE; mobilegym-rl has a separate matching Apache-2.0 license and retains vendored notices.", "declared": "Apache-2.0", "retrieved_at": "2026-10-17T11:22:59Z", "url": "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/LICENSE" }, "data": { "basis": "Limited to maintainers’ dataset compilation/curation and authored synthetic content; third-party user content remains subject to platform terms and its authors’ rights.", "declared": "CC-BY-NC-4.0", "retrieved_at": "2026-10-17T11:22:59Z", "url": "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/LICENSE-DATA" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:18:19Z", "role": "primary", "sha256": "815294dd2aa4bf8264978c3bfedee85a3a2c835c511860c732314a4c282a974a", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:27:49Z", "role": "primary", "sha256": "0ecef460fecfd8aa91dcbb2cd4cca17b033f620f85a50d7f0b598ed19a27c0eb", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/bench_env/splits/train.txt" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:27:48Z", "role": "primary", "sha256": "a3d485015d480f4f1ebbd12dbbf02c25c0aa667d1afcf5f6fa35f0263a82c567", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/Purewhiter/mobilegym/HEAD/bench_env/splits/test.txt" } ] }, { "id": "github:openmobile2/mobilegym-plusplus", "name": "MobileGym++ / MobileGym++ Bench", "url": "https://github.com/OpenMobile2/MobileGym-plusplus", "environments": [ "web" ], "profile": { "actions": [ "GUI gestures", "foreground-app tool calls in hybrid mode" ], "counts": [ { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/OpenMobile2/MobileGym-plusplus/HEAD/bench_env/benchmark/official.json", "population": "official.json", "qualifier": "exact", "unit": "task definitions", "value": 215 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/OpenMobile2/MobileGym-plusplus/HEAD/README.md", "population": "complete simulator installation", "qualifier": "exact", "unit": "simulated clients", "value": 63 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/OpenMobile2/MobileGym-plusplus/HEAD/README.md", "population": "new clients in this release", "qualifier": "exact", "unit": "simulated clients", "value": 35 } ], "evaluation": "Programmatic checkers inspect changes in application state; both modes share task definitions and initial states.", "evidence_urls": [ "https://raw.githubusercontent.com/OpenMobile2/MobileGym-plusplus/HEAD/README.md", "https://raw.githubusercontent.com/OpenMobile2/MobileGym-plusplus/HEAD/bench_env/benchmark/official.json" ], "focus": [ "task completion", "cross-application workflows", "GUI and app-native tool comparison" ], "notes": "The task count was checked against unique task_id values in the published manifest. The 35 new clients are a subset of 63 total clients. This simulator runs in a browser; Android-emulator apps are a separate part of OpenMobile-2.", "observations": [ "screenshots" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-17T01:52:51Z", "url": "https://raw.githubusercontent.com/OpenMobile2/MobileGym-plusplus/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T01:48:10Z", "role": "primary", "sha256": "d9f850ec196e715650e679cb81515adaa64ddc5a60a882bfe97b0f6a098ddc83", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/OpenMobile2/MobileGym-plusplus/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T01:50:11Z", "role": "primary", "sha256": "35c1641b349201c68917fbe4c4036814487678e9588d9b1c903af26917800316", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/OpenMobile2/MobileGym-plusplus/HEAD/bench_env/benchmark/official.json" } ] }, { "id": "github:jylee425/mobilesafetybench", "name": "MobileSafetyBench", "url": "https://github.com/jylee425/mobilesafetybench", "environments": [ "android" ], "profile": { "actions": [ "tap/long-tap/swipe", "navigation buttons and text editing", "refuse, ask-consent and complete", "task-specific shortcut actions" ], "counts": [], "evaluation": "Appium-backed task evaluators track goal achievement and harm prevention as separate booleans, plus risk-detection step and completion progress. The base evaluator defaults to max_step=15; task-specific overrides must be retained. Results are not a single generic success-rate metric.", "evidence_urls": [ "https://raw.githubusercontent.com/jylee425/mobilesafetybench/HEAD/README.md", "https://api.github.com/repos/jylee425/mobilesafetybench/git/trees/release/ver.3?recursive=1", "https://raw.githubusercontent.com/jylee425/mobilesafetybench/HEAD/mobile_safety/environment.py", "https://raw.githubusercontent.com/jylee425/mobilesafetybench/HEAD/mobile_safety/component/parser.py", "https://raw.githubusercontent.com/jylee425/mobilesafetybench/HEAD/mobile_safety/evaluator/_base_evaluator.py" ], "focus": [ "mobile task helpfulness", "harm prevention during Android interaction" ], "notes": "Inspected release/ver.3 branch exposes task definitions, Android environment and per-task evaluators. It requires Android API 34, Appium/ADB and documented third-party app versions; those APKs/accounts were not acquired. The action parser includes semantic shortcuts in addition to screen gestures, so this is not a strictly coordinate-only action setting. No corpus task total is inferred from evaluator-module or file counts. Apache-2.0 applies to repository code, not automatically app/data assets.", "observations": [ "screenshot pixels", "parsed UI hierarchy text", "task instruction and interaction history" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:58Z", "url": "https://raw.githubusercontent.com/jylee425/mobilesafetybench/HEAD/LICENSE.txt" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "e733115d9b2fbe910cb3891162f04a89af8cbda25255c8f0bf60f78101e4309a", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/jylee425/mobilesafetybench/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:14:05Z", "role": "primary", "sha256": "8dc66f6c5087a32c5c9fb7b3104ea6f9e0082255e2ba3cf9ba8244ce5a24da6f", "supports": [ "details.benchmark_protocol" ], "url": "https://api.github.com/repos/jylee425/mobilesafetybench/git/trees/release/ver.3?recursive=1" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:09Z", "role": "primary", "sha256": "f56d6345546a9c67a2113cfba5b7148f5bd6b48fac5c34f1a5f4080e2d81ae2f", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/jylee425/mobilesafetybench/HEAD/mobile_safety/environment.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:10Z", "role": "primary", "sha256": "120487c3a6492371321dd3eebcd65dc59de6a82454d622d7fed3a7d86b8fa6c5", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/jylee425/mobilesafetybench/HEAD/mobile_safety/component/parser.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:23:14Z", "role": "primary", "sha256": "918e3aa64ab3f578fb54580da123c1ff4508022b64ce74f88ce28c7a612756a4", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/jylee425/mobilesafetybench/HEAD/mobile_safety/evaluator/_base_evaluator.py" } ] }, { "id": "github:tongyi-mai/mobileworld", "name": "MobileWorld (Tongyi-MAI)", "url": "https://github.com/Tongyi-MAI/MobileWorld", "environments": [ "android" ], "profile": { "actions": [ "Android GUI operations", "MCP calls when enabled", "agent-user interaction when enabled" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/Tongyi-MAI/MobileWorld/HEAD/README.md", "population": "README benchmark overview", "qualifier": "exact", "unit": "tasks", "value": 201 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/Tongyi-MAI/MobileWorld/HEAD/README.md", "population": "README benchmark overview", "qualifier": "exact", "unit": "mobile applications", "value": 20 } ], "evaluation": "The publisher documents text-answer matching, backend database verification, local-storage inspection and application callbacks.", "evidence_urls": [ "https://raw.githubusercontent.com/Tongyi-MAI/MobileWorld/HEAD/README.md" ], "focus": [ "Android task completion", "agent-user interaction", "hybrid GUI and MCP use" ], "notes": "Publisher-reported inventory. GUI-only, agent-user and MCP-augmented configurations have different dependencies and task eligibility; preserve enabled modes and step budget with results.", "observations": [ "Android application interface", "user-agent interaction when enabled" ], "setting": "interactive" }, "licenses": { "code": { "basis": "Repository code license; self-hosted applications, model APIs and MCP services keep their own terms.", "declared": "Apache-2.0", "retrieved_at": "2026-10-17T09:14:09Z", "url": "https://raw.githubusercontent.com/Tongyi-MAI/MobileWorld/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T09:12:16Z", "role": "primary", "sha256": "fbc306b3414dfdb230171358efd6eb8d67573fbb94e11487a4a7e971e2abf0d9", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details" ], "url": "https://raw.githubusercontent.com/Tongyi-MAI/MobileWorld/HEAD/README.md" } ] }, { "id": "url:849a86beaee006c6b147", "name": "OmniAct", "url": "https://huggingface.co/datasets/Writer/omniact", "environments": [ "desktop", "web" ], "profile": { "actions": [ "PyAutoGUI script: pointer movement/click/drag, key presses, hotkeys, and text entry" ], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/Writer/omniact/raw/5e4bcb26745afc8d58ba9519ea9c9fb832386d26/README.md", "population": "train split, HF revision 5e4bcb26", "qualifier": "exact", "unit": "task examples", "value": 6788 }, { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/Writer/omniact/raw/5e4bcb26745afc8d58ba9519ea9c9fb832386d26/README.md", "population": "test split, HF revision 5e4bcb26", "qualifier": "exact", "unit": "task examples", "value": 2020 }, { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/Writer/omniact/raw/5e4bcb26745afc8d58ba9519ea9c9fb832386d26/README.md", "population": "val split, HF revision 5e4bcb26", "qualifier": "exact", "unit": "task examples", "value": 991 } ], "evaluation": "The bundled evaluator reports weighted sequence match and action match, comparing action types and applying coordinate, key, and writing penalties. Target element-box metadata is evaluation-only. The retained scorer has a click-penalty defect: predicted coordinates are read from gold_line, so score reproduction requires explicit scorer-version review.", "evidence_urls": [ "https://huggingface.co/datasets/Writer/omniact/raw/5e4bcb26745afc8d58ba9519ea9c9fb832386d26/README.md", "https://huggingface.co/api/datasets/Writer/omniact", "https://huggingface.co/datasets/Writer/omniact/raw/5e4bcb26745afc8d58ba9519ea9c9fb832386d26/eval.py" ], "focus": [ "offline desktop/web action-script generation" ], "notes": "This released split table totals 9,799 examples; it is not silently substituted for any paper-level collection size. Public files include train/test/val JSON indexes, task/image archives, and eval.py. Evaluation is offline script comparison, not execution success on a live desktop. MIT is the Hub data declaration; original app/content rights are not resolved by it.", "observations": [ "screenshot", "natural-language task" ], "setting": "offline" }, "licenses": { "code": null, "data": { "basis": "publisher model/dataset card", "declared": "mit", "retrieved_at": "2026-10-16T06:38:27Z", "url": "https://huggingface.co/datasets/Writer/omniact" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:38:27Z", "role": "primary", "sha256": "2db0a5dd2b983dded3b90a8ecd66cadbb39074f03778a339822418531661b3e3", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://huggingface.co/api/datasets/Writer/omniact" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:10:00Z", "role": "primary", "sha256": "7b54ad557f89d621fe39c3d2fe9f02164a2bfcb7fef0db610d964586d3673693", "supports": [ "details.benchmark_protocol" ], "url": "https://huggingface.co/datasets/Writer/omniact/raw/5e4bcb26745afc8d58ba9519ea9c9fb832386d26/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:22:06Z", "role": "primary", "sha256": "a786de678af46df5e2b6b1a2a69d2ee239fe8fd5b63663765d84d39e21fc2365", "supports": [ "details.benchmark_protocol" ], "url": "https://huggingface.co/datasets/Writer/omniact/raw/5e4bcb26745afc8d58ba9519ea9c9fb832386d26/eval.py" }, { "method": "huggingface_api", "retrieved_at": "2026-10-16T06:38:27Z", "role": "primary", "sha256": "2db0a5dd2b983dded3b90a8ecd66cadbb39074f03778a339822418531661b3e3", "supports": [ "details", "licenses.data" ], "url": "https://huggingface.co/api/datasets/Writer/omniact" } ] }, { "id": "github:osu-nlp-group/online-mind2web", "name": "Online Mind2Web", "url": "https://github.com/OSU-NLP-Group/Online-Mind2Web", "environments": [ "web" ], "profile": { "actions": [ "web interactions" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/OSU-NLP-Group/Online-Mind2Web/HEAD/README.md", "population": "documented Online-Mind2Web suite", "qualifier": "exact", "unit": "tasks", "value": 300 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/OSU-NLP-Group/Online-Mind2Web/HEAD/README.md", "population": "documented site coverage", "qualifier": "exact", "unit": "websites", "value": 136 } ], "evaluation": "Human evaluation and WebJudge-based evaluation are documented as distinct assessment paths.", "evidence_urls": [ "https://raw.githubusercontent.com/OSU-NLP-Group/Online-Mind2Web/HEAD/README.md" ], "focus": [ "live-web task completion" ], "notes": "Tasks are actively replaced when websites or requirements become invalid. Record the task revision and submission schema; a constant headline task count does not imply an unchanged test set.", "observations": [], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/OSU-NLP-Group/Online-Mind2Web/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "49d7cc317fd2ba0df65abf65a0357d1fa2384e59019a96d4e24b784888599eaa", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol", "details.leaderboard_url" ], "url": "https://raw.githubusercontent.com/OSU-NLP-Group/Online-Mind2Web/HEAD/README.md" } ] }, { "id": "github:facebookresearch/openapps", "name": "OpenApps", "url": "https://github.com/facebookresearch/OpenApps", "environments": [ "web" ], "profile": { "actions": [ "click", "type", "scroll", "configured BrowserGym/AgentLab actions" ], "counts": [], "evaluation": "Tasks define check_if_task_is_complete(initial_state,current_state); app-specific reward logic evaluates underlying state, with custom checks for code-editor file contents. Observation/history flags are configurable. No universal single fixed task total or published-run reproduction is implied.", "evidence_urls": [ "https://raw.githubusercontent.com/facebookresearch/OpenApps/HEAD/README.md", "https://raw.githubusercontent.com/facebookresearch/OpenApps/HEAD/docs/tasks.md", "https://raw.githubusercontent.com/facebookresearch/OpenApps/HEAD/docs/agents.md", "https://api.github.com/repos/facebookresearch/OpenApps/git/trees/main?recursive=1" ], "focus": [ "UI-agent reliability under environment variation", "state-based browser task evaluation" ], "notes": "Preserve the code/tag, app state, theme/layout/content choices, goal variant and action/observation configuration. Current main refactors the paper-era appearance surface, and the v1.0-paper tag includes post-publication fixes that may move rewards. The environments run as browser apps; phone/tablet viewports are not native Android/iOS environments.", "observations": [ "screenshots", "accessibility trees", "set-of-marks and coordinates (configurable)" ], "setting": "interactive" }, "licenses": { "code": { "basis": "Reviewed repository license declaration for code; third-party dependencies, data, weights and executable environment images are separate.", "declared": "CC-BY-NC-4.0", "retrieved_at": "2026-10-18T04:35:20Z", "url": "https://raw.githubusercontent.com/facebookresearch/OpenApps/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:35:18Z", "role": "primary", "sha256": "1df0f66584b37e7dba90811a7732d3b0c13c96cd7da4aec3970a18be36a6cec1", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/facebookresearch/OpenApps/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:35:18Z", "role": "primary", "sha256": "34aa76fbd6cca6ac2f3949792144662b90d85d7a18988afabcb570167ea5c1d6", "supports": [ "identity", "release_status", "details.repository_tree_sha", "details.benchmark_protocol" ], "url": "https://api.github.com/repos/facebookresearch/OpenApps/git/trees/main?recursive=1" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:40:40Z", "role": "primary", "sha256": "af030b93a7604c063b08f003b0a0124a6e95e560649d0306bfe40dd87fed563d", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/facebookresearch/OpenApps/HEAD/docs/tasks.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:40:40Z", "role": "primary", "sha256": "aede4dfd965bd3484336f40ddb1bd6db48aed00ab45e27c048b76cc4601d71b3", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/facebookresearch/OpenApps/HEAD/docs/agents.md" } ] }, { "id": "url:c73ba0618d1b3c505887", "name": "OS-Critic Bench", "url": "https://huggingface.co/datasets/OS-Copilot/OS-Critic-Bench", "environments": [ "cross-platform", "desktop", "web" ], "profile": { "actions": [ "critic-model responses" ], "counts": [ { "basis": "computed", "evidence_url": "https://huggingface.co/datasets/OS-Copilot/OS-Critic-Bench/raw/main/README.md", "population": "mobile/desktop/web configurations in the Hub card", "qualifier": "exact", "unit": "platform JSONL splits", "value": 3 } ], "evaluation": "The official repository runs model inference and computes accuracy with cal_acc.py.", "evidence_urls": [ "https://raw.githubusercontent.com/numbmelon/OS-Oracle/HEAD/README.md", "https://huggingface.co/datasets/OS-Copilot/OS-Critic-Bench/raw/main/README.md" ], "focus": [ "GUI critic prediction" ], "notes": "No sample count is inferred from the n<1K Hub size band. Evaluation code availability does not establish the release of OS-Oracle training data or weights.", "observations": [ "benchmark JSONL", "UI screenshots" ], "setting": "offline" }, "licenses": { "code": null, "data": { "basis": "Publisher dataset-card license declaration for OS-Critic-Bench; not an evaluation-code or model-weight license.", "declared": "apache-2.0", "retrieved_at": "2026-10-17T11:23:07Z", "url": "https://huggingface.co/datasets/OS-Copilot/OS-Critic-Bench/raw/main/README.md" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:18:18Z", "role": "primary", "sha256": "58305b8ebe1af5dfbddd636484d63722d4e21d28ed3e22bb8c0068e654f33479", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/numbmelon/OS-Oracle/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:23:07Z", "role": "primary", "sha256": "4139e25e8911ca8450f6db9acb8dc3a98c126c3c43514c72bcefa4ed33b8ffa8", "supports": [ "licenses.data", "details.benchmark_protocol" ], "url": "https://huggingface.co/datasets/OS-Copilot/OS-Critic-Bench/raw/main/README.md" } ] }, { "id": "github:tml-epfl/os-harm", "name": "OS-Harm", "url": "https://github.com/tml-epfl/os-harm", "environments": [ "desktop" ], "profile": { "actions": [ "OSWorld/PyAutoGUI desktop interaction" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/README.md", "population": "reported benchmark total; not a recount of unique released task IDs", "qualifier": "exact", "unit": "tasks as reported by the paper/README", "value": 150 }, { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misuse.json", "population": "released misuse task-ID list; before injection/goal expansion", "qualifier": "exact", "unit": "task IDs", "value": 50 }, { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_injection.json", "population": "released injection task-ID list; before injection/goal expansion", "qualifier": "exact", "unit": "task IDs", "value": 10 }, { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misbehavior.json", "population": "released misbehavior task-ID list; before injection/goal expansion", "qualifier": "exact", "unit": "task IDs", "value": 50 } ], "evaluation": "Automated LLM judge scores both task accuracy and safety from saved execution traces. Keep judge model/prompt version and harm category with a result; the scripts separately compute predicted metrics. This is an interactive desktop safety benchmark, not a single offline grounding score.", "evidence_urls": [ "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/README.md", "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/run.py", "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misuse.json", "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_injection.json", "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misbehavior.json" ], "focus": [ "deliberate misuse", "prompt-injection safety", "unintended harmful behavior during normal tasks" ], "notes": "README/paper reports 150 tasks, whereas retained misuse/injection/misbehavior lists contain 50/10/50 IDs. run.py expands injection vectors/goals, so the released ID inventory is not a direct recount of paper settings; the complete expanded total was not reconstructed. Judge agreement F1 values in the README measure agreement with human labels, not agent success. Experiments use OSWorld desktop apps, with VMware documented as tested and screenshot/a11y/SOM observation choices. Code Apache-2.0 does not establish VM/app/data rights. Publisher asks that the benchmark not be used directly as agent training data.", "observations": [ "screenshot", "accessibility tree", "screenshot plus accessibility tree", "set-of-marks" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T09:39:34Z", "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T09:36:33Z", "role": "primary", "sha256": "d001158f58ac37b01c8470224db1567a52feb8010877305f7966637cf7400706", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details", "details.benchmark_protocol", "notes" ], "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:23:14Z", "role": "primary", "sha256": "86dd59a252fd654fc3244268a75de37b2c7f97e16f6b1de8c4cf885458a422f7", "supports": [ "details.benchmark_protocol", "notes" ], "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/run.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:22:04Z", "role": "primary", "sha256": "8ff6092a51cc924c5225ace01c011c6481a2461eb0a47b7360eddaf6619b898c", "supports": [ "details.benchmark_protocol", "notes" ], "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misuse.json" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:22:04Z", "role": "primary", "sha256": "b4c451236bb4ff36eb7e790dfa8cb517bffb245602bc441080314b13c8aba54f", "supports": [ "details.benchmark_protocol", "notes" ], "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_injection.json" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:22:05Z", "role": "primary", "sha256": "8eb6bade72696c3c0b53160a05b006a773c9e553ab5e401418b54d9acc98c0f2", "supports": [ "details.benchmark_protocol", "notes" ], "url": "https://raw.githubusercontent.com/tml-epfl/os-harm/HEAD/evaluation_examples/test_misbehavior.json" } ] }, { "id": "github:os-copilot/os-map", "name": "OS-Map", "url": "https://github.com/OS-Copilot/OS-Map", "environments": [ "desktop", "linux" ], "profile": { "actions": [ "actions executed by the OSWorld-derived DesktopEnv" ], "counts": [ { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/OS-Copilot/OS-Map/HEAD/evaluation_examples/all.json", "population": "evaluation_examples/all.json", "qualifier": "exact", "unit": "category/task references", "value": 416 } ], "evaluation": "Task configurations provide evaluators for application/system state, using an OSWorld-derived runtime.", "evidence_urls": [ "https://raw.githubusercontent.com/OS-Copilot/OS-Map/HEAD/README.md", "https://raw.githubusercontent.com/OS-Copilot/OS-Map/HEAD/evaluation_examples/all.json", "https://raw.githubusercontent.com/OS-Copilot/OS-Map/HEAD/evaluation_examples/README.md" ], "focus": [ "breadth of desktop task coverage", "depth of desktop task execution" ], "notes": "416 category/task pairs are unique, but only 412 bare ID strings are unique across categories. This is a manifest-reference count, not a claim of 416 semantically distinct tasks. Main runtime documentation specifies Ubuntu; the subdirectory README retains older Windows-only wording.", "observations": [ "desktop observations from an Ubuntu virtual machine" ], "setting": "interactive" }, "licenses": { "code": { "basis": "Repository code LICENSE; no license is inferred for the VM image or bundled third-party applications.", "declared": "MIT", "retrieved_at": "2026-10-17T09:14:07Z", "url": "https://raw.githubusercontent.com/OS-Copilot/OS-Map/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T09:12:14Z", "role": "primary", "sha256": "fc9f3c8d9fffdff8b51e312b013be0d9fdfc10e57fdc2ec4a8a8d012c4b42475", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details" ], "url": "https://raw.githubusercontent.com/OS-Copilot/OS-Map/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T09:17:22Z", "role": "primary", "sha256": "4199af4c8bc60451079d8a2e7598ab7386bd82b0d5c9ad7e6927d6cb8e03ad6f", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/OS-Copilot/OS-Map/HEAD/evaluation_examples/all.json" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T09:17:22Z", "role": "primary", "sha256": "a6b60d5721cfdd26416e7d0ae3192360896427a97ccb3793812cc150e9cd7126", "supports": [ "details.benchmark_protocol", "notes" ], "url": "https://raw.githubusercontent.com/OS-Copilot/OS-Map/HEAD/evaluation_examples/README.md" } ] }, { "id": "github:xlang-ai/osworld", "name": "OSWorld", "url": "https://github.com/xlang-ai/OSWorld", "environments": [ "desktop", "linux", "windows", "macos" ], "profile": { "actions": [ "mouse and keyboard via PyAutoGUI" ], "counts": [ { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/xlang-ai/OSWorld/HEAD/evaluation_examples/test_all.json", "population": "test_all.json", "qualifier": "exact", "unit": "task definitions", "value": 369 }, { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/xlang-ai/OSWorld/HEAD/evaluation_examples/test_nogdrive.json", "population": "test_nogdrive.json", "qualifier": "exact", "unit": "task definitions", "value": 361 } ], "evaluation": "Task-specific execution-based evaluators inspect application or system state.", "evidence_urls": [ "https://raw.githubusercontent.com/xlang-ai/OSWorld/HEAD/README.md", "https://export.arxiv.org/api/query?id_list=2403.03186%2C2403.03788%2C2403.07718%2C2403.10170%2C2403.11905%2C2403.16971%2C2403.17918%2C2403.20271%2C2404.01549%2C2404.01744%2C2404.02475%2C2404.03648%2C2404.05427%2C2404.05719%2C2404.05955%2C2404.06474%2C2404.07972%2C2404.08755%2C2404.08860%2C2404.08948%2C2404.09992%2C2404.10887%2C2404.11459%2C2404.16006%2C2404.16048%2C2404.16054%2C2404.16660%2C2404.18074%2C2404.19296%2C2405.00516%2C2405.11120%2C2405.12842%2C2405.14573%2C2405.14751%2C2405.15793%2C2405.16247%2C2405.20309%2C2406.01014%2C2406.01623%2C2406.03679%2C2406.06947%2C2406.07822%2C2406.08184%2C2406.08340%2C2406.08451%2C2406.10227%2C2406.10819%2C2406.11317%2C2406.11896%2C2406.12373&max_results=50", "https://raw.githubusercontent.com/xlang-ai/OSWorld/HEAD/evaluation_examples/test_all.json", "https://raw.githubusercontent.com/xlang-ai/OSWorld/HEAD/evaluation_examples/test_nogdrive.json" ], "focus": [ "task completion", "cross-application workflows" ], "notes": "Counts were computed from unique task identifiers in the two inspected manifests. The subsets differ; record the manifest, repository revision, agent interface, and step budget with a score.", "observations": [ "screenshots" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/xlang-ai/OSWorld/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "83b8684c2f4ac4f31996b8b4752195fcb6cc5429f89d92398177b9b721d787fc", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol", "details.leaderboard_url" ], "url": "https://raw.githubusercontent.com/xlang-ai/OSWorld/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:31:28Z", "role": "primary", "sha256": "be244bfae7f89ef9449b69d85aac44295eb7b5f7e5e1bbedd76df312696676a7", "supports": [ "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?id_list=2403.03186%2C2403.03788%2C2403.07718%2C2403.10170%2C2403.11905%2C2403.16971%2C2403.17918%2C2403.20271%2C2404.01549%2C2404.01744%2C2404.02475%2C2404.03648%2C2404.05427%2C2404.05719%2C2404.05955%2C2404.06474%2C2404.07972%2C2404.08755%2C2404.08860%2C2404.08948%2C2404.09992%2C2404.10887%2C2404.11459%2C2404.16006%2C2404.16048%2C2404.16054%2C2404.16660%2C2404.18074%2C2404.19296%2C2405.00516%2C2405.11120%2C2405.12842%2C2405.14573%2C2405.14751%2C2405.15793%2C2405.16247%2C2405.20309%2C2406.01014%2C2406.01623%2C2406.03679%2C2406.06947%2C2406.07822%2C2406.08184%2C2406.08340%2C2406.08451%2C2406.10227%2C2406.10819%2C2406.11317%2C2406.11896%2C2406.12373&max_results=50" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T11:34:59Z", "role": "primary", "sha256": "9ebc5187cbd727ef26c24626820076b102fff812863c640a67c467fea9542ab5", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/xlang-ai/OSWorld/HEAD/evaluation_examples/test_all.json" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T11:34:59Z", "role": "primary", "sha256": "fcb9497e93a8986407345d3012c872c9c2fed253420730fbea64ccfcace67dbb", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/xlang-ai/OSWorld/HEAD/evaluation_examples/test_nogdrive.json" } ] }, { "id": "github:xlang-ai/osworld-g", "name": "OSWorld-G", "url": "https://github.com/xlang-ai/OSWorld-G", "environments": [ "desktop" ], "profile": { "actions": [ "click point or box center", "refusal represented by negative coordinates in the evaluator" ], "counts": [], "evaluation": "The inspected Jedi runner maps predictions from resized-image coordinates back to the original image, then checks the predicted center against rectangle/polygon targets or the refusal rule. It reports overall and classification-group accuracy. This is offline grounding, not OSWorld task-execution success.", "evidence_urls": [ "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/README.md", "https://api.github.com/repos/xlang-ai/OSWorld-G/git/trees/main?recursive=1", "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/evaluation/qwen25_vllm_osworld_g_jedi.py", "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/evaluation/eval.py" ], "focus": [ "desktop GUI grounding", "infeasible-target refusal" ], "notes": "The repository retains original OSWorld-G.json and refined OSWorld-G_refined.json instructions; treat them as alternative benchmark versions, not additional disjoint tasks. Refined instructions reduce requirements beyond grounding. The same repository also contains Jedi training/rendering material, which is excluded from benchmark counts. The standalone generic eval.py entry point is a stub; the named model runner supplies image_size to its geometry helper. No task count or successful execution is inferred from repository file totals. Apache-2.0 is recorded for code only.", "observations": [ "screenshot", "instruction" ], "setting": "offline" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "e814851bc3016a2abf59358f99b80afd9c2bd5894fc34baddfb7e4d0aa03bd45", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:14:05Z", "role": "primary", "sha256": "48308bba4f26b02cb7b8fbbc433bb31ec81c494dabd3af437ddbb10b8d9ce667", "supports": [ "details.benchmark_protocol" ], "url": "https://api.github.com/repos/xlang-ai/OSWorld-G/git/trees/main?recursive=1" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:22:04Z", "role": "primary", "sha256": "64f64e0b847e0fa200aa6051e20b8035a436df00e2ef0d769688ab13fa4d3046", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/evaluation/qwen25_vllm_osworld_g_jedi.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:09Z", "role": "primary", "sha256": "b85f5685ce6c200ea648471666177859693d7f40cf50c12d08765ae68580de4a", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/evaluation/eval.py" } ] }, { "id": "github:wuklab/osworld-human", "name": "OSWorld-Human", "url": "https://github.com/WukLab/osworld-human", "environments": [ "desktop" ], "profile": { "actions": [ "efficiency scores" ], "counts": [], "evaluation": "The scorer consumes OSWorld results and a specified maximum step budget; the README distinguishes single-action and grouped-action WES+ and WES-.", "evidence_urls": [ "https://raw.githubusercontent.com/WukLab/osworld-human/HEAD/README.md" ], "focus": [ "action efficiency relative to humans" ], "notes": "Reference solutions are published for evaluation and expressly discouraged for training. No task count is inferred from file count.", "observations": [ "OSWorld result trajectories", "human reference actions" ], "setting": "offline" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T03:59:54Z", "role": "primary", "sha256": "35c2f4dbed9d9bfc57f4c5a3f2fd70d313d6b463403b482c2fd4066d27b02cc1", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/WukLab/osworld-human/HEAD/README.md" } ] }, { "id": "github:microsoft/precision-cua-bench", "name": "Precision CUA Bench", "url": "https://github.com/microsoft/precision-cua-bench", "environments": [ "cross-platform" ], "profile": { "actions": [ "pixel (x,y) cursor coordinate" ], "counts": [], "evaluation": "The inspected evaluator scales target boxes from a 1000×1000 reference grid into image pixels and tests pixel predictions, allowing ±10 pixels around the center along any target dimension narrower than 1 pixel after scaling. It reports cumulative hit accuracy by turn (earlier hits remain correct), miss distances, any-turn hit rate, and a correction rate whose denominator is all successfully processed samples. The default budget is five turns; retain prompt/feedback variant, turn budget, and processed/skipped coverage.", "evidence_urls": [ "https://raw.githubusercontent.com/microsoft/precision-cua-bench/HEAD/README.md", "https://raw.githubusercontent.com/microsoft/precision-cua-bench/HEAD/eval/test_allturns_api.py", "https://api.github.com/repos/microsoft/precision-cua-bench/git/trees/main?recursive=1" ], "focus": [ "pixel-precise editor cursor placement", "multi-turn visual self-correction" ], "notes": "The release has four editor/theme JSONL partitions (Cursor/VS Code × dark/light) and screenshot files. Partitions are not train/test splits or independent task totals. README’s initial field table calls bbox pixel coordinates, but the evaluation-format section and actual ratio_h/ratio_w=dimension/1000 code use normalized target boxes; profile follows the inspected evaluator and retains the inconsistency. Source/images were not executed or downloaded as a dataset. MIT is recorded for repository code only.", "observations": [ "code-editor screenshot", "cursor-location instruction", "red-cross feedback marking previous incorrect prediction" ], "setting": "offline" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:41:59Z", "url": "https://raw.githubusercontent.com/microsoft/precision-cua-bench/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:59Z", "role": "primary", "sha256": "84ad4fb798abb776dce813cf4dab296d6e03248d8ae29388e1f06cccd27d1241", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/microsoft/precision-cua-bench/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:08Z", "role": "primary", "sha256": "1fda7c3980fe7a7c373478eda6ecfabff227e92c2b1f19c6fe903b408f583797", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/microsoft/precision-cua-bench/HEAD/eval/test_allturns_api.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:14:06Z", "role": "primary", "sha256": "ede84e3dda47f80fada4234caa2423db08e21ff7667a37599c03aae657b2f84b", "supports": [ "details.benchmark_protocol" ], "url": "https://api.github.com/repos/microsoft/precision-cua-bench/git/trees/main?recursive=1" } ] }, { "id": "github:mcgill-nlp/safearena", "name": "SafeArena", "url": "https://github.com/McGill-NLP/safearena", "environments": [ "web" ], "profile": { "actions": [ "web interactions" ], "counts": [ { "basis": "reported", "evidence_url": "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=700&max_results=100&sortBy=submittedDate&sortOrder=descending", "population": "safe split", "qualifier": "exact", "reference_url": "https://arxiv.org/abs/2503.04957", "unit": "tasks", "value": 250 }, { "basis": "reported", "evidence_url": "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=700&max_results=100&sortBy=submittedDate&sortOrder=descending", "population": "harmful split", "qualifier": "exact", "reference_url": "https://arxiv.org/abs/2503.04957", "unit": "tasks", "value": 250 } ], "evaluation": "Task outcomes and the Agent Risk Assessment categories evaluate agent behavior on benign and harmful requests.", "evidence_urls": [ "https://raw.githubusercontent.com/McGill-NLP/safearena/HEAD/README.md", "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=700&max_results=100&sortBy=submittedDate&sortOrder=descending" ], "focus": [ "web-agent safety", "task completion" ], "notes": "Use SafeArena-specific containers rather than ordinary WebArena instances. The documented task splits require access approval; the profile does not imply unrestricted dataset access.", "observations": [], "setting": "interactive" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T09:36:33Z", "role": "primary", "sha256": "ae50121aa60ac26b32d8d6ad6f578f9091a81a3cb9ca28b98050bddee16c929a", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol", "details.leaderboard_url" ], "url": "https://raw.githubusercontent.com/McGill-NLP/safearena/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:36:54Z", "role": "primary", "sha256": "4953a8e0f029016aef668bce98cec4d2873a7ae49c1fe33c4311fa5eb8d24089", "supports": [ "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=700&max_results=100&sortBy=submittedDate&sortOrder=descending" } ] }, { "id": "github:os-copilot/scienceboard", "name": "ScienceBoard", "url": "https://github.com/OS-Copilot/ScienceBoard", "environments": [ "desktop" ], "profile": { "actions": [ "PyAutoGUI desktop actions", "code/tool commands and answer/stop actions according to configured agent mode" ], "counts": [ { "basis": "computed", "evidence_url": "https://api.github.com/repos/OS-Copilot/ScienceBoard/git/trees/main?recursive=1", "population": "tasks/VM/**/*.json at tree 21b5eb0f; excludes tasks/Raw and tasks/Dual", "qualifier": "exact", "unit": "VM task definitions", "value": 169 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/OS-Copilot/ScienceBoard/HEAD/docs/verifier-audit.md", "population": "VM suite: Celestia, ChimeraX, GrassGIS, KAlgebra, Lean, TeXstudio", "qualifier": "exact", "unit": "application families", "value": 6 } ], "evaluation": "Application-specific verifiers inspect state, files, database values, equations, compilation or answer/stop conditions. The official audit records 242 verifier entries across its VM task suite. This profile covers the released VM task definitions; Docker smoke results and Raw/Dual alternatives are separate populations.", "evidence_urls": [ "https://raw.githubusercontent.com/OS-Copilot/ScienceBoard/HEAD/README.md", "https://raw.githubusercontent.com/OS-Copilot/ScienceBoard/HEAD/docs/verifier-audit.md", "https://raw.githubusercontent.com/OS-Copilot/ScienceBoard/HEAD/docs/known-issues.md", "https://api.github.com/repos/OS-Copilot/ScienceBoard/git/trees/main?recursive=1", "https://raw.githubusercontent.com/OS-Copilot/ScienceBoard/HEAD/sci/base/task.py" ], "focus": [ "scientific-software desktop workflows", "GUI and code-assisted scientific tasks" ], "notes": "Manifest count 169 matches the official verifier audit. Other Raw/Dual definitions can represent alternative execution modes of the same problems and are not added. Official Docker migration documents a 20-task acceptance subset, not a full-suite performance result; legacy VMware remains. External service sign-in/rate limits, amd64 binaries and ChimeraX redistribution terms limit portability. Only code MIT is established; VM, scientific data and bundled apps need their own terms.", "observations": [ "desktop screenshots", "accessibility-tree observations", "set-of-marks with accessibility tree" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:41:59Z", "url": "https://raw.githubusercontent.com/OS-Copilot/ScienceBoard/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "4c3fe93aa740dbf4a279a246ddd836f2590a2a36f39b820abe166e38cb484577", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/OS-Copilot/ScienceBoard/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:10Z", "role": "primary", "sha256": "d276e11127abc5826bd1b2aa2caf543fcd45a8a60477774e8c8b1054067997ec", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/OS-Copilot/ScienceBoard/HEAD/docs/verifier-audit.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:10Z", "role": "primary", "sha256": "4f90270617c0c1d58f523c0a7f2e4603fa67a49560d68e2f88cfa080d764313b", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/OS-Copilot/ScienceBoard/HEAD/docs/known-issues.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:14:07Z", "role": "primary", "sha256": "3398670a517a5d1d71f420dd866493d3f8a77853e3933fd4928103f9a8be1e4f", "supports": [ "details.benchmark_protocol" ], "url": "https://api.github.com/repos/OS-Copilot/ScienceBoard/git/trees/main?recursive=1" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:35:24Z", "role": "primary", "sha256": "d6a1481b453c074e34739fd29def56efa35a1dc10e856418f557f067e9ff08c5", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/OS-Copilot/ScienceBoard/HEAD/sci/base/task.py" } ] }, { "id": "url:7abb418096432f56c685", "name": "ScreenSpot", "url": "https://huggingface.co/datasets/rootsautomation/ScreenSpot", "environments": [ "cross-platform" ], "profile": { "actions": [ "predicted click point; a predicted box is reduced to its center" ], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/bevaya/ScreenSpot/raw/0be08781e2e188582f6131625ae1598d443b4d5d/README.md", "population": "Hub test split at 0be08781; not unique screenshots", "qualifier": "exact", "unit": "instruction–target examples", "value": 1272 } ], "evaluation": "The SeeClick evaluator scores whether the predicted point falls inside the target box, with platform and text/icon breakdowns. The Hub packaging uses xyxy boxes, whereas the original evaluator reads xywh and converts it; adapt the loader to the chosen release.", "evidence_urls": [ "https://huggingface.co/datasets/bevaya/ScreenSpot/raw/0be08781e2e188582f6131625ae1598d443b4d5d/README.md", "https://huggingface.co/api/datasets/rootsautomation/ScreenSpot", "https://raw.githubusercontent.com/njucckevin/SeeClick/HEAD/pretrain/screenspot_test.py" ], "focus": [ "single-step GUI element grounding" ], "notes": "Offline, zero-shot evaluation across mobile, desktop, and web. The card explicitly excludes training use in its intended-use description. Public API reports ungated Parquet files and canonical Hub id bevaya/ScreenSpot; the catalog identity is preserved. Apache-2.0 is the Hub data declaration. No runtime or underlying screenshot rights audit was performed.", "observations": [ "screenshot", "English instruction" ], "setting": "offline" }, "licenses": { "code": null, "data": { "basis": "publisher model/dataset card", "declared": "apache-2.0", "retrieved_at": "2026-10-16T06:38:26Z", "url": "https://huggingface.co/datasets/rootsautomation/ScreenSpot" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:38:26Z", "role": "primary", "sha256": "9e2d2a2c076032225911ae1032b548eafa6331ea81a2779632610f0f1c82dfd9", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://huggingface.co/api/datasets/rootsautomation/ScreenSpot" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:10:00Z", "role": "primary", "sha256": "5ab2240c064502f75a6726311e10e7724ab7055347697f883b15a6ee8e7d00a6", "supports": [ "details.benchmark_protocol" ], "url": "https://huggingface.co/datasets/bevaya/ScreenSpot/raw/0be08781e2e188582f6131625ae1598d443b4d5d/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:22:04Z", "role": "primary", "sha256": "eb72113eefd4868114f8a8105eba2d3385da9034a5b3a4860661e03ae7dcae15", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/njucckevin/SeeClick/HEAD/pretrain/screenspot_test.py" }, { "method": "huggingface_api", "retrieved_at": "2026-10-16T06:38:26Z", "role": "primary", "sha256": "9e2d2a2c076032225911ae1032b548eafa6331ea81a2779632610f0f1c82dfd9", "supports": [ "details", "licenses.data" ], "url": "https://huggingface.co/api/datasets/rootsautomation/ScreenSpot" }, { "method": "huggingface_repository_identity", "retrieved_at": "2026-10-17T19:14:56Z", "role": "primary", "sha256": "af0c4babeff84158ff196b54bf47242923bbf502d2805a1dd4eb52dba44be89c", "supports": [ "aliases" ], "url": "https://huggingface.co/api/datasets/rootsautomation/ScreenSpot" } ] }, { "id": "github:likaixin2000/screenspot-pro-gui-grounding", "name": "ScreenSpot Pro", "url": "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding", "environments": [ "desktop", "windows", "macos", "linux" ], "profile": { "actions": [ "predicted target coordinates" ], "counts": [ { "basis": "computed", "evidence_url": "https://huggingface.co/datasets/likaixin/ScreenSpot-Pro/raw/main/README.md", "population": "sum of icon/text counts in the published card table", "qualifier": "exact", "unit": "target annotations", "value": 1581 } ], "evaluation": "For positive targets, the predicted point must fall inside the annotated bounding box.", "evidence_urls": [ "https://raw.githubusercontent.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding/HEAD/readme.md", "https://raw.githubusercontent.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding/HEAD/eval_screenspot_pro.py", "https://huggingface.co/datasets/likaixin/ScreenSpot-Pro/raw/main/README.md" ], "focus": [ "GUI grounding" ], "notes": "Language, instruction style, and positive/negative target configuration are explicit evaluator options. This profile describes target localization, not end-to-end workflow completion. The target count is computed from the card table, not from expanded language/style evaluation instances.", "observations": [ "high-resolution screenshots", "target instructions" ], "setting": "offline" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "7b64ccad9cfec24c2c9cb35f55923bff93d53729cef3cc7463c79b29bdb1de1a", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol", "details.leaderboard_url" ], "url": "https://raw.githubusercontent.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding/HEAD/readme.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T11:38:05Z", "role": "primary", "sha256": "a77c859775b33591b5619e8520c94702a53a1f9a29ada93e66744dbe23afe9dc", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding/HEAD/eval_screenspot_pro.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T11:35:55Z", "role": "primary", "sha256": "76ae2231564fe84af029f8625c02d9cd7fb6cf85c5f8af1b2a2593e2107719d9", "supports": [ "details.benchmark_protocol", "environments" ], "url": "https://huggingface.co/datasets/likaixin/ScreenSpot-Pro/raw/main/README.md" } ] }, { "id": "url:bd7214de6d98917c2350", "name": "ScreenSpot v2", "url": "https://huggingface.co/datasets/OS-Copilot/ScreenSpot-v2", "environments": [ "cross-platform" ], "profile": { "actions": [ "click coordinates" ], "counts": [], "evaluation": "The inspected Jedi evaluation adapter reads the v2 mobile/desktop/web annotation files, scores a point against the target box, and reports overall, platform, and text/icon accuracy. This is a named compatible evaluator implementation; the OS-Atlas dataset card itself supplies no evaluator specification.", "evidence_urls": [ "https://huggingface.co/api/datasets/OS-Copilot/ScreenSpot-v2", "https://osatlas.github.io/", "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/evaluation/qwen25_vllm_screenspot_v2_jedi.py", "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/evaluation/eval.py" ], "focus": [ "single-step GUI grounding on the ScreenSpot-v2 release" ], "notes": "The public Hub revision 5efbb1f1 contains three platform JSON annotation files and an image ZIP; its card only declares Apache-2.0. No sample total or equivalence to the original ScreenSpot test count is asserted. Keep v2 separate from v1 and retain evaluator/version and coordinate conversion when comparing results. Files were listed, not downloaded as data.", "observations": [ "screenshot", "instruction" ], "setting": "offline" }, "licenses": { "code": null, "data": { "basis": "publisher model/dataset card", "declared": "apache-2.0", "retrieved_at": "2026-10-16T06:38:26Z", "url": "https://huggingface.co/datasets/OS-Copilot/ScreenSpot-v2" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:38:26Z", "role": "primary", "sha256": "7feddadc306e384f8df6a3d9c8afc0c776cb3734c4c7fe3eb842eab85537f408", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://huggingface.co/api/datasets/OS-Copilot/ScreenSpot-v2" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:22:05Z", "role": "primary", "sha256": "94547cb88756cf6a36bfb86d5fe9bdca1e40d7edb9e452c33cb462bbe760fed0", "supports": [ "details.benchmark_protocol" ], "url": "https://osatlas.github.io/" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:09Z", "role": "primary", "sha256": "0e8772bc8fd5a25c50d3c2f6b0caee05ed2ab4659d46d99d60adf2b5ab29e997", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/evaluation/qwen25_vllm_screenspot_v2_jedi.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:21:09Z", "role": "primary", "sha256": "b85f5685ce6c200ea648471666177859693d7f40cf50c12d08765ae68580de4a", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/xlang-ai/OSWorld-G/HEAD/evaluation/eval.py" }, { "method": "huggingface_api", "retrieved_at": "2026-10-16T06:38:26Z", "role": "primary", "sha256": "7feddadc306e384f8df6a3d9c8afc0c776cb3734c4c7fe3eb842eab85537f408", "supports": [ "details", "licenses.data" ], "url": "https://huggingface.co/api/datasets/OS-Copilot/ScreenSpot-v2" } ] }, { "id": "url:17729e1c532ba7043404", "name": "ScreenSpot-v2 instruction variants", "url": "https://huggingface.co/datasets/likaixin/ScreenSpot-v2-variants", "environments": [ "cross-platform" ], "profile": { "actions": [ "target localization", "identification of impossible requests" ], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/likaixin/ScreenSpot-v2-variants/raw/main/README.md", "population": "instruction-variant design", "qualifier": "exact", "unit": "instruction types", "value": 4 } ], "evaluation": "Publisher supplies original, action, description and negative instruction types; no common end-to-end success metric is asserted.", "evidence_urls": [ "https://huggingface.co/datasets/likaixin/ScreenSpot-v2-variants/raw/main/README.md", "https://huggingface.co/api/datasets/likaixin/ScreenSpot-v2-variants" ], "focus": [ "instruction-form robustness", "negative grounding requests" ], "notes": "Reuses ScreenSpot-v2 imagery. Image-file count is not treated as a count of independent screenshots or tasks.", "observations": [ "screenshots", "instructions" ], "setting": "offline" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T04:07:19Z", "role": "primary", "sha256": "7fb2bc9499d8a2b37b663c116af49d11bba493e6767d76a5305fad93257bebf2", "supports": [ "name", "kind", "summary", "topics", "scope", "relations", "details" ], "url": "https://huggingface.co/datasets/likaixin/ScreenSpot-v2-variants/raw/main/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T04:07:19Z", "role": "primary", "sha256": "dea98efa75e063eee2d6bfb5d46da94b41df3d779e2fb0bdaea7866534b5c469", "supports": [ "identity", "release_status", "details", "environments" ], "url": "https://huggingface.co/api/datasets/likaixin/ScreenSpot-v2-variants" } ] }, { "id": "github:ai-agents-2030/spa-bench", "name": "SPA-Bench", "url": "https://github.com/ai-agents-2030/SPA-Bench", "environments": [ "android" ], "profile": { "actions": [ "agent-specific Android actions" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md", "population": "complete README task collection", "qualifier": "exact", "unit": "tasks", "value": 340 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md", "population": "single-app subset", "qualifier": "exact", "unit": "tasks", "value": 300 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md", "population": "cross-app subset", "qualifier": "exact", "unit": "tasks", "value": 40 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md", "population": "README total including third-party, Google, and system apps", "qualifier": "exact", "unit": "applications", "value": 66 } ], "evaluation": "A multimodal evaluator and annotated key-component matching assess completion; the framework also reports resource-use metrics.", "evidence_urls": [ "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md", "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/Documentation.md" ], "focus": [ "mobile task completion", "resource use", "cross-app workflows" ], "notes": "The two task subsets partition the 340 tasks. The integrated agent and evaluator modes must accompany results. The preconfigured emulator snapshot is still marked coming soon in the inspected documentation.", "observations": [ "screenshot trajectories" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:41:58Z", "url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "5294a67728072a3658cb2073bdf9a084fe3c00d87a848923dff68adb35213072", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "5294a67728072a3658cb2073bdf9a084fe3c00d87a848923dff68adb35213072", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:03:23Z", "role": "primary", "sha256": "a55290d62f8823097c73d74bac4eb08853a3caff724382e60a6a9364e7ced987", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/ai-agents-2030/SPA-Bench/HEAD/Documentation.md" } ] }, { "id": "github:xlang-ai/spider2-v", "name": "Spider2-V", "url": "https://github.com/xlang-ai/Spider2-V", "environments": [ "web", "desktop" ], "profile": { "actions": [ "PyAutoGUI mouse and keyboard actions" ], "counts": [ { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/xlang-ai/Spider2-V/HEAD/evaluation_examples/test_all.json", "population": "evaluation_examples/test_all.json", "qualifier": "exact", "unit": "task definitions", "value": 494 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/xlang-ai/Spider2-V/HEAD/README.md", "population": "README task-suite coverage", "qualifier": "exact", "unit": "applications", "value": 20 } ], "evaluation": "Task-specific evaluation functions are invoked after agent execution.", "evidence_urls": [ "https://raw.githubusercontent.com/xlang-ai/Spider2-V/HEAD/README.md", "https://raw.githubusercontent.com/xlang-ai/Spider2-V/HEAD/evaluation_examples/test_all.json" ], "focus": [ "data-science workflows", "multi-application task completion" ], "notes": "Counted unique task IDs in the full manifest. Verbose versus abstract instructions and account versus non-account tasks form different subsets. The Ubuntu VM and external-account requirements are part of the protocol; no accounts or environments were provisioned.", "observations": [ "screenshots", "accessibility trees", "set-of-marks views" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:59Z", "url": "https://raw.githubusercontent.com/xlang-ai/Spider2-V/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "e3edf30ab45da16382b46d8c542607c1bbbfcf077b6df57a2decf938fc84ff0f", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/xlang-ai/Spider2-V/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:58Z", "role": "primary", "sha256": "e3edf30ab45da16382b46d8c542607c1bbbfcf077b6df57a2decf938fc84ff0f", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/xlang-ai/Spider2-V/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:06:30Z", "role": "primary", "sha256": "338dd38ba7047b0cf6dbc68f3f7b9374f962708fa2dadcc8d126c232dfc31746", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/xlang-ai/Spider2-V/HEAD/evaluation_examples/test_all.json" } ] }, { "id": "github:theagentcompany/theagentcompany", "name": "TheAgentCompany", "url": "https://github.com/TheAgentCompany/TheAgentCompany", "environments": [ "web", "desktop" ], "profile": { "actions": [ "web browsing", "code execution", "coworker communication" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/TheAgentCompany/TheAgentCompany/HEAD/README.md", "population": "workspace images documented by the README", "qualifier": "exact", "unit": "task images", "value": 175 } ], "evaluation": "Task-specific evaluator scripts run within the prepared workplace environment.", "evidence_urls": [ "https://raw.githubusercontent.com/TheAgentCompany/TheAgentCompany/HEAD/README.md" ], "focus": [ "professional workflows", "multi-tool tasks" ], "notes": "Each task is packaged as a Docker image. Record task-image and server versions; the environment includes simulated coworkers and external application state.", "observations": [], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T10:48:42Z", "url": "https://raw.githubusercontent.com/TheAgentCompany/TheAgentCompany/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T10:42:34Z", "role": "primary", "sha256": "d1646f67d49d38ef1e81bf7b5057dfc3d6f172b4c8a7413a2cc78741ccb08723", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol", "details.leaderboard_url" ], "url": "https://raw.githubusercontent.com/TheAgentCompany/TheAgentCompany/HEAD/README.md" } ] }, { "id": "github:web-arena-x/visualwebarena", "name": "VisualWebArena", "url": "https://github.com/web-arena-x/visualwebarena", "environments": [ "web" ], "profile": { "actions": [ "browser element actions" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/web-arena-x/visualwebarena/HEAD/README.md", "population": "full VWA suite", "qualifier": "exact", "unit": "tasks", "value": 910 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/web-arena-x/visualwebarena/HEAD/README.md", "population": "one task from each template type", "qualifier": "exact", "unit": "human trajectories", "value": 233 } ], "evaluation": "Execution-based completion checks for visually grounded web tasks.", "evidence_urls": [ "https://raw.githubusercontent.com/web-arena-x/visualwebarena/HEAD/README.md" ], "focus": [ "multimodal web task completion" ], "notes": "The 233 demonstration trajectories are not the full 910-task evaluation set. Observation modes and captioning components differ across baseline agents.", "observations": [ "screenshots with set-of-mark annotations", "caption-augmented accessibility tree" ], "setting": "interactive" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "182529c2fce1e41f3a2162727205c54da8b3d2381b2273d726170ee9d8cf65cc", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/web-arena-x/visualwebarena/HEAD/README.md" } ] }, { "id": "github:charlypoly/web-context-bench-python", "name": "Web Context Bench (Python)", "url": "https://github.com/charlypoly/web-context-bench-python", "environments": [ "web" ], "profile": { "actions": [ "READ text answer", "ACT selector, text/label, accessible role/name, DOM index, Stagehand reference or screenshot point, according to the representation" ], "counts": [], "evaluation": "READ uses normalized exact match against an answer or allowed alias. ACT resolves predicted identifiers to ground-truth elements, or checks screenshot points inside the target box. Representation-token accounting subtracts fixed-prompt tokens; screenshots use billed input usage minus the fixed prompt. These are answer/target correctness and token metrics, not completed-task success.", "evidence_urls": [ "https://raw.githubusercontent.com/charlypoly/web-context-bench-python/58d604bfaaac60ad42f02013f53f9fb301c1dcda/README.md", "https://raw.githubusercontent.com/charlypoly/web-context-bench-python/58d604bfaaac60ad42f02013f53f9fb301c1dcda/METHODOLOGY.md", "https://raw.githubusercontent.com/charlypoly/web-context-bench-python/58d604bfaaac60ad42f02013f53f9fb301c1dcda/src/wpbench/scoring.py" ], "focus": [ "Browser-representation token use", "Factual READ accuracy", "ACT target-element selection" ], "notes": "Fixed local-page observations; the reported run uses claude-sonnet-5 and a shared prompt, without scrolling or an agent loop. Browser Use disables its viewport cutoff and sends untruncated text; Stagehand uses a separate matched browser load. Short-string representability labels are approximate. Offline does not mean API-free. Third-party snapshot redistribution remains unresolved; results were not reproduced.", "observations": [ "Top-document HTML", "Markdown converted from HTML", "Playwright accessibility snapshot", "Browser Use indexed DOM", "Stagehand hybrid snapshot", "Viewport screenshot" ], "setting": "offline" }, "licenses": { "code": { "basis": "Complete MIT license text in the repository root at the inspected commit; repository code only.", "declared": "MIT", "retrieved_at": "2026-10-20T11:51:41.499735Z", "url": "https://raw.githubusercontent.com/charlypoly/web-context-bench-python/58d604bfaaac60ad42f02013f53f9fb301c1dcda/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-20T11:51:30.874397Z", "role": "primary", "sha256": "14775f6e3daba69d4684380fd33bf717bdc3093b65be796eb863bbd27d0c0bab", "supports": [ "name", "kind", "summary", "topics", "scope", "notes", "environments", "details.benchmark_protocol.focus", "details.benchmark_protocol.setting", "details.benchmark_protocol.observations", "details.benchmark_protocol.actions", "details.benchmark_protocol.notes" ], "url": "https://raw.githubusercontent.com/charlypoly/web-context-bench-python/58d604bfaaac60ad42f02013f53f9fb301c1dcda/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-20T11:52:30.131541Z", "role": "primary", "sha256": "1c3c0cbf9f9d9f7cd9e7a2f027ecb217fd5c16fa157916281a15bb7f403f6f1c", "supports": [ "summary", "environments", "topics", "scope", "notes", "licenses.data", "details.benchmark_protocol.focus", "details.benchmark_protocol.setting", "details.benchmark_protocol.observations", "details.benchmark_protocol.actions", "details.benchmark_protocol.evaluation", "details.benchmark_protocol.notes" ], "url": "https://raw.githubusercontent.com/charlypoly/web-context-bench-python/58d604bfaaac60ad42f02013f53f9fb301c1dcda/METHODOLOGY.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-20T11:53:28.567861Z", "role": "primary", "sha256": "8cada5d71b5a61e7aff2e1ac2b951c266eb2db7c082c06c5a13aa127d42b3fc5", "supports": [ "summary", "topics", "scope", "notes", "details.benchmark_protocol.evaluation", "details.benchmark_protocol.notes" ], "url": "https://raw.githubusercontent.com/charlypoly/web-context-bench-python/58d604bfaaac60ad42f02013f53f9fb301c1dcda/src/wpbench/scoring.py" } ] }, { "id": "github:web-arena-x/webarena", "name": "WebArena", "url": "https://github.com/web-arena-x/webarena", "environments": [ "web" ], "profile": { "actions": [ "browser element actions" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/web-arena-x/webarena/HEAD/README.md", "population": "canonical WebArena evaluation suite", "qualifier": "exact", "unit": "task examples", "value": 812 } ], "evaluation": "Functional correctness of completed web tasks.", "evidence_urls": [ "https://raw.githubusercontent.com/web-arena-x/webarena/HEAD/README.md", "https://export.arxiv.org/api/query?id_list=1802.08802%2C1808.09132%2C2005.03776%2C2101.09465%2C2103.16057%2C2104.08560%2C2105.13231%2C2107.03374%2C2107.13731%2C2108.03353%2C2202.02312%2C2202.08137%2C2204.01691%2C2205.11029%2C2207.01206%2C2209.08199%2C2209.08655%2C2209.14927%2C2209.15099%2C2210.02663%2C2210.03629%2C2210.03945%2C2211.07615%2C2301.10165%2C2301.13280%2C2302.04761%2C2303.07280%2C2303.08774%2C2303.17491%2C2303.17580%2C2304.04370%2C2304.07061%2C2305.04091%2C2305.05662%2C2305.08144%2C2305.10601%2C2305.11854%2C2305.12487%2C2305.14257%2C2305.14325%2C2305.16504%2C2305.16653%2C2305.19308%2C2306.00245%2C2306.06070%2C2306.07863%2C2307.10088%2C2307.12856%2C2307.13854%2C2308.01990&max_results=50" ], "focus": [ "web task completion" ], "notes": "The canonical repository documents self-hosted websites and environment resets. Its README recommends BrowserGym/AgentLab for newer infrastructure; harness changes must be recorded.", "observations": [ "HTML or accessibility tree" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/web-arena-x/webarena/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "081c826497f9b7da4952771ee128c2215ec3532379db0fe9f7fcbd899d9c9287", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol", "details.leaderboard_url" ], "url": "https://raw.githubusercontent.com/web-arena-x/webarena/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:31:20Z", "role": "primary", "sha256": "cbe8bfcbce4170a94a6cf42de01fbe0a448e7b1d29fa75a0de877105a98ec08e", "supports": [ "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?id_list=1802.08802%2C1808.09132%2C2005.03776%2C2101.09465%2C2103.16057%2C2104.08560%2C2105.13231%2C2107.03374%2C2107.13731%2C2108.03353%2C2202.02312%2C2202.08137%2C2204.01691%2C2205.11029%2C2207.01206%2C2209.08199%2C2209.08655%2C2209.14927%2C2209.15099%2C2210.02663%2C2210.03629%2C2210.03945%2C2211.07615%2C2301.10165%2C2301.13280%2C2302.04761%2C2303.07280%2C2303.08774%2C2303.17491%2C2303.17580%2C2304.04370%2C2304.07061%2C2305.04091%2C2305.05662%2C2305.08144%2C2305.10601%2C2305.11854%2C2305.12487%2C2305.14257%2C2305.14325%2C2305.16504%2C2305.16653%2C2305.19308%2C2306.00245%2C2306.06070%2C2306.07863%2C2307.10088%2C2307.12856%2C2307.13854%2C2308.01990&max_results=50" } ] }, { "id": "url:5e2bd1b3398974b79c9c", "name": "WebClick", "url": "https://huggingface.co/datasets/Hcompany/WebClick", "environments": [ "web" ], "profile": { "actions": [ "click target or bounding box" ], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/Hcompany/WebClick/raw/main/README.md", "population": "test split", "qualifier": "exact", "unit": "examples", "value": 1639 } ], "evaluation": "Assess localization against annotated targets; agentbrowse, humanbrowse and calendars are distinct source buckets.", "evidence_urls": [ "https://huggingface.co/datasets/Hcompany/WebClick/raw/main/README.md" ], "focus": [ "instruction-conditioned web localization" ], "notes": "The card reports more than 100 websites but does not provide a precise website total. Grounding accuracy does not establish task completion.", "observations": [ "web screenshots", "instructions" ], "setting": "offline" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T04:01:33Z", "role": "primary", "sha256": "958bbf0eb9f61afd6c576673fbe6f7d84e6c4b8a9d4d8c4fc0e1224b5d8da30b", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://huggingface.co/datasets/Hcompany/WebClick/raw/main/README.md" } ] }, { "id": "github:mcgill-nlp/weblinx", "name": "WebLINX", "url": "https://github.com/mcgill-nlp/weblinx", "environments": [ "web" ], "profile": { "actions": [ "predicted browser actions" ], "counts": [ { "basis": "reported", "evidence_url": "https://export.arxiv.org/api/query?id_list=2308.03688%2C2308.15272%2C2309.08172%2C2309.11436%2C2310.04716%2C2310.04869%2C2310.05126%2C2310.06770%2C2310.08740%2C2310.10634%2C2310.11441%2C2310.16042%2C2310.18331%2C2311.01767%2C2311.07562%2C2311.09278%2C2311.12983%2C2311.18751%2C2312.03003%2C2312.06677%2C2312.08914%2C2312.11671%2C2312.13108%2C2312.13771%2C2312.15820%2C2312.16062%2C2401.01614%2C2401.04124%2C2401.07781%2C2401.10935%2C2401.13178%2C2401.13649%2C2401.13919%2C2401.14268%2C2401.16158%2C2402.01030%2C2402.04476%2C2402.04615%2C2402.05930%2C2402.06596%2C2402.07456%2C2402.07939%2C2402.07945%2C2402.08567%2C2402.11941%2C2402.14073%2C2402.15057%2C2402.16965%2C2402.17553%2C2403.02713&max_results=50", "population": "original paper benchmark", "qualifier": "approximately", "unit": "interactions", "value": 100000 }, { "basis": "reported", "evidence_url": "https://export.arxiv.org/api/query?id_list=2308.03688%2C2308.15272%2C2309.08172%2C2309.11436%2C2310.04716%2C2310.04869%2C2310.05126%2C2310.06770%2C2310.08740%2C2310.10634%2C2310.11441%2C2310.16042%2C2310.18331%2C2311.01767%2C2311.07562%2C2311.09278%2C2311.12983%2C2311.18751%2C2312.03003%2C2312.06677%2C2312.08914%2C2312.11671%2C2312.13108%2C2312.13771%2C2312.15820%2C2312.16062%2C2401.01614%2C2401.04124%2C2401.07781%2C2401.10935%2C2401.13178%2C2401.13649%2C2401.13919%2C2401.14268%2C2401.16158%2C2402.01030%2C2402.04476%2C2402.04615%2C2402.05930%2C2402.06596%2C2402.07456%2C2402.07939%2C2402.07945%2C2402.08567%2C2402.11941%2C2402.14073%2C2402.15057%2C2402.16965%2C2402.17553%2C2403.02713&max_results=50", "population": "original paper benchmark", "qualifier": "approximately", "unit": "expert demonstrations", "value": 2300 }, { "basis": "reported", "evidence_url": "https://export.arxiv.org/api/query?id_list=2308.03688%2C2308.15272%2C2309.08172%2C2309.11436%2C2310.04716%2C2310.04869%2C2310.05126%2C2310.06770%2C2310.08740%2C2310.10634%2C2310.11441%2C2310.16042%2C2310.18331%2C2311.01767%2C2311.07562%2C2311.09278%2C2311.12983%2C2311.18751%2C2312.03003%2C2312.06677%2C2312.08914%2C2312.11671%2C2312.13108%2C2312.13771%2C2312.15820%2C2312.16062%2C2401.01614%2C2401.04124%2C2401.07781%2C2401.10935%2C2401.13178%2C2401.13649%2C2401.13919%2C2401.14268%2C2401.16158%2C2402.01030%2C2402.04476%2C2402.04615%2C2402.05930%2C2402.06596%2C2402.07456%2C2402.07939%2C2402.07945%2C2402.08567%2C2402.11941%2C2402.14073%2C2402.15057%2C2402.16965%2C2402.17553%2C2403.02713&max_results=50", "population": "original paper benchmark", "qualifier": "more-than", "unit": "websites", "value": 150 } ], "evaluation": "Offline evaluation compares action predictions against recorded human demonstrations.", "evidence_urls": [ "https://raw.githubusercontent.com/mcgill-nlp/weblinx/HEAD/README.md", "https://export.arxiv.org/api/query?id_list=2308.03688%2C2308.15272%2C2309.08172%2C2309.11436%2C2310.04716%2C2310.04869%2C2310.05126%2C2310.06770%2C2310.08740%2C2310.10634%2C2310.11441%2C2310.16042%2C2310.18331%2C2311.01767%2C2311.07562%2C2311.09278%2C2311.12983%2C2311.18751%2C2312.03003%2C2312.06677%2C2312.08914%2C2312.11671%2C2312.13108%2C2312.13771%2C2312.15820%2C2312.16062%2C2401.01614%2C2401.04124%2C2401.07781%2C2401.10935%2C2401.13178%2C2401.13649%2C2401.13919%2C2401.14268%2C2401.16158%2C2402.01030%2C2402.04476%2C2402.04615%2C2402.05930%2C2402.06596%2C2402.07456%2C2402.07939%2C2402.07945%2C2402.08567%2C2402.11941%2C2402.14073%2C2402.15057%2C2402.16965%2C2402.17553%2C2403.02713&max_results=50", "https://huggingface.co/datasets/McGill-NLP/WebLINX/raw/main/README.md" ], "focus": [ "conversational web navigation", "next-action prediction" ], "notes": "The figures are rounded descriptions from the original paper. WebLINX 1.1 / BrowserGym removes some demonstrations and changes evaluated steps, including tab actions; do not mix those results or counts with the original release. This profile does not treat replayed demonstration steps as live task completion.", "observations": [ "dialogue history", "HTML", "screenshots", "action history" ], "setting": "offline" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/mcgill-nlp/weblinx/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "15c3f1656afd861894b180bca7d9214b4cf78a6ad9cd0601c80f5d67d2839528", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/mcgill-nlp/weblinx/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "15c3f1656afd861894b180bca7d9214b4cf78a6ad9cd0601c80f5d67d2839528", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/mcgill-nlp/weblinx/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:31:24Z", "role": "primary", "sha256": "3aac6e9748769efff3c475ede0da192776daa03431d231b5f67e2aa50f8a3be9", "supports": [ "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?id_list=2308.03688%2C2308.15272%2C2309.08172%2C2309.11436%2C2310.04716%2C2310.04869%2C2310.05126%2C2310.06770%2C2310.08740%2C2310.10634%2C2310.11441%2C2310.16042%2C2310.18331%2C2311.01767%2C2311.07562%2C2311.09278%2C2311.12983%2C2311.18751%2C2312.03003%2C2312.06677%2C2312.08914%2C2312.11671%2C2312.13108%2C2312.13771%2C2312.15820%2C2312.16062%2C2401.01614%2C2401.04124%2C2401.07781%2C2401.10935%2C2401.13178%2C2401.13649%2C2401.13919%2C2401.14268%2C2401.16158%2C2402.01030%2C2402.04476%2C2402.04615%2C2402.05930%2C2402.06596%2C2402.07456%2C2402.07939%2C2402.07945%2C2402.08567%2C2402.11941%2C2402.14073%2C2402.15057%2C2402.16965%2C2402.17553%2C2403.02713&max_results=50" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:03:23Z", "role": "primary", "sha256": "9bd7512e257f5914d66b091530d7eb711a6f90bc21dc89cbba53015c7136c527", "supports": [ "details.benchmark_protocol" ], "url": "https://huggingface.co/datasets/McGill-NLP/WebLINX/raw/main/README.md" } ] }, { "id": "url:ac6d0f80765f5a32255a", "name": "WebRewardBench", "url": "https://huggingface.co/datasets/LangAGI-Lab/WebRewardBench", "environments": [ "web" ], "profile": { "actions": [ "candidate-decision preference" ], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/LangAGI-Lab/WebRewardBench/raw/main/README.md", "population": "Hub test split", "qualifier": "exact", "unit": "rows", "value": 776 } ], "evaluation": "Compare reward-model preferences with the provided chosen/rejected decision labels; no independent score was reproduced.", "evidence_urls": [ "https://huggingface.co/datasets/LangAGI-Lab/WebRewardBench/raw/main/README.md", "https://raw.githubusercontent.com/kyle8581/Web-Shepherd/HEAD/README.md" ], "focus": [ "web process-reward evaluation" ], "notes": "The current Hub row count and project case-count claim use unresolved counting units. Candidate lists mean a row must not be assumed to equal one task or one pair.", "observations": [ "page screenshots", "text observations", "action history", "task checklists" ], "setting": "offline" }, "licenses": { "code": null, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:44:50Z", "role": "primary", "sha256": "0968f60eea10b95f79cd16ad3746b0e0b1acaeb0e97c45856f709d53a14dd73a", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://huggingface.co/datasets/LangAGI-Lab/WebRewardBench/raw/main/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:44:04Z", "role": "primary", "sha256": "07431058d8b9a71b80e75b7a74a40a8346c1503eec240628459cc4d4e2f0802f", "supports": [ "summary", "environments" ], "url": "https://raw.githubusercontent.com/kyle8581/Web-Shepherd/HEAD/README.md" } ] }, { "id": "github:princeton-nlp/webshop", "name": "WebShop", "url": "https://github.com/princeton-nlp/WebShop", "environments": [ "web" ], "profile": { "actions": [ "search queries", "click actions", "product option selection" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/princeton-nlp/WebShop/HEAD/README.md", "population": "crowd-sourced instruction collection", "qualifier": "exact", "unit": "instructions", "value": 12087 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/princeton-nlp/WebShop/HEAD/README.md", "population": "full product inventory", "qualifier": "approximately", "unit": "products", "value": 1180000 } ], "evaluation": "", "evidence_urls": [ "https://raw.githubusercontent.com/princeton-nlp/WebShop/HEAD/README.md" ], "focus": [ "instruction-conditioned product selection", "web navigation" ], "notes": "This is a simulated shop. The default preview loads 1,000 products, while the full inventory is much larger; report the loaded subset and observation mode. Instructions and products are different populations. Detailed reward semantics are not normalized in this profile.", "observations": [ "HTML pages", "simplified text views" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:41:58Z", "url": "https://raw.githubusercontent.com/princeton-nlp/WebShop/HEAD/LICENSE.md" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "79b7a90a1413a52deb53142b5bbd81ecf40f61743bd4601c2835e91375898ec0", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/princeton-nlp/WebShop/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "79b7a90a1413a52deb53142b5bbd81ecf40f61743bd4601c2835e91375898ec0", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/princeton-nlp/WebShop/HEAD/README.md" } ] }, { "id": "url:3b6dfd52f98bfe76bc7f", "name": "WebTailBench", "url": "https://huggingface.co/datasets/microsoft/WebTailBench", "environments": [ "web" ], "profile": { "actions": [ "browser interaction" ], "counts": [ { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/microsoft/WebTailBench/raw/main/README.md", "population": "main V1/V2 suite (shared identities)", "qualifier": "exact", "unit": "task identities", "value": 609 }, { "basis": "reported", "evidence_url": "https://huggingface.co/datasets/microsoft/WebTailBench/raw/main/README.md", "population": "refusal suite", "qualifier": "exact", "unit": "tasks", "value": 111 } ], "evaluation": "Task-specific precomputed rubrics; refusal tasks are a separate evaluation population.", "evidence_urls": [ "https://huggingface.co/datasets/microsoft/WebTailBench/raw/main/README.md" ], "focus": [ "live web task completion", "unsafe-request refusal" ], "notes": "V2 refreshes task strings and rubrics while preserving V1 task IDs. Counts are task definitions, not independently verified runnable website states.", "observations": [ "web pages" ], "setting": "interactive" }, "licenses": { "code": null, "data": { "basis": "publisher model/dataset card metadata; third-party assets retain their own terms", "declared": "mit", "retrieved_at": "2026-10-17T02:42:32Z", "url": "https://huggingface.co/datasets/microsoft/WebTailBench/raw/main/README.md" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:42:32Z", "role": "primary", "sha256": "c4cfc361d1d47d7855269af4110483363964e23f9c65244a1182830eccccbb7f", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details", "licenses.data" ], "url": "https://huggingface.co/datasets/microsoft/WebTailBench/raw/main/README.md" } ] }, { "id": "github:friedrichor/webtestbench", "name": "WebTestBench", "url": "https://github.com/friedrichor/WebTestBench", "environments": [ "web" ], "profile": { "actions": [ "browser interaction", "test checklist generation", "defect reports" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/friedrichor/WebTestBench/HEAD/README.md", "population": "publisher benchmark overview", "qualifier": "exact", "unit": "application categories", "value": 7 }, { "basis": "computed", "evidence_url": "https://huggingface.co/api/datasets/friedrichor/WebTestBench", "population": "Hub file manifest", "qualifier": "exact", "unit": "web-application archives", "value": 100 } ], "evaluation": "The supplied WebTester pipeline separates checklist generation from interactive defect detection and provides a scoring workflow.", "evidence_urls": [ "https://raw.githubusercontent.com/friedrichor/WebTestBench/HEAD/README.md", "https://huggingface.co/api/datasets/friedrichor/WebTestBench", "https://huggingface.co/datasets/friedrichor/WebTestBench/raw/main/README.md" ], "focus": [ "web functionality testing", "latent permission and business constraints" ], "notes": "The 100 archive files are not counted as 100 independent test tasks or defects. Backend/tool versions and deployment settings affect reproducibility.", "observations": [ "web application interface", "development instruction" ], "setting": "interactive" }, "licenses": { "code": { "basis": "Repository source LICENSE; external tools and services retain their own licenses.", "declared": "Apache-2.0", "retrieved_at": "2026-10-17T11:22:55Z", "url": "https://raw.githubusercontent.com/friedrichor/WebTestBench/HEAD/LICENSE" }, "data": { "basis": "Publisher Hub-card license declaration for the released benchmark dataset.", "declared": "apache-2.0", "retrieved_at": "2026-10-17T11:23:08Z", "url": "https://huggingface.co/datasets/friedrichor/WebTestBench/raw/main/README.md" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:18:18Z", "role": "primary", "sha256": "b41472026cd7d4f81a01d58bf39193dc1fabc7875c61e370090a20cc77500cdd", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/friedrichor/WebTestBench/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:23:08Z", "role": "primary", "sha256": "db1e97f40107038ae03835dc20f17b13b37e74b5c8587f8c39b441271618f017", "supports": [ "identity", "release_status", "details.benchmark_protocol" ], "url": "https://huggingface.co/api/datasets/friedrichor/WebTestBench" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T11:23:08Z", "role": "primary", "sha256": "6f2f94a25ed66a14779f6b4e6686044d0510eb32a0187645cc473978d93f7c5b", "supports": [ "licenses.data", "details.benchmark_protocol" ], "url": "https://huggingface.co/datasets/friedrichor/WebTestBench/raw/main/README.md" } ] }, { "id": "github:minorjerry/webvoyager", "name": "WebVoyager", "url": "https://github.com/MinorJerry/WebVoyager", "environments": [ "web" ], "profile": { "actions": [ "Selenium-backed browser interactions" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/MinorJerry/WebVoyager/HEAD/README.md", "population": "WebVoyager task collection", "qualifier": "exact", "unit": "task queries", "value": 643 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/MinorJerry/WebVoyager/HEAD/README.md", "population": "WebVoyager task collection", "qualifier": "exact", "unit": "websites", "value": 15 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/MinorJerry/WebVoyager/HEAD/README.md", "population": "separate GAIA validation-derived web subset", "qualifier": "exact", "unit": "tasks", "value": 90 } ], "evaluation": "The repository supplies an automated multimodal-model evaluation protocol.", "evidence_urls": [ "https://raw.githubusercontent.com/MinorJerry/WebVoyager/HEAD/README.md" ], "focus": [ "live-web task completion" ], "notes": "The GAIA-derived subset is separate from the 643 WebVoyager queries. Live sites and time-sensitive travel tasks can change; record the task snapshot and evaluation date.", "observations": [ "webpage screenshots", "textual task instructions" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/MinorJerry/WebVoyager/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "b5523941b2773634fdcf359b5b89bf0aca589f33ad9db30692c8b1912ed5dda5", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/MinorJerry/WebVoyager/HEAD/README.md" } ] }, { "id": "github:microsoft/windowsagentarena", "name": "Windows Agent Arena", "url": "https://github.com/microsoft/WindowsAgentArena", "environments": [ "windows", "desktop" ], "profile": { "actions": [ "agent-defined Windows interactions" ], "counts": [ { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/microsoft/WindowsAgentArena/HEAD/src/win-arena-container/client/evaluation_examples_windows/test_all.json", "population": "evaluation_examples_windows/test_all.json", "qualifier": "exact", "unit": "task definitions", "value": 154 } ], "evaluation": "Task configuration selects an evaluator and expected application or system outcome; evaluators return a reward for the resulting state.", "evidence_urls": [ "https://raw.githubusercontent.com/microsoft/WindowsAgentArena/HEAD/README.md", "https://raw.githubusercontent.com/microsoft/WindowsAgentArena/HEAD/docs/Develop-Tasks.md", "https://raw.githubusercontent.com/microsoft/WindowsAgentArena/HEAD/src/win-arena-container/client/evaluation_examples_windows/test_all.json" ], "focus": [ "Windows task completion", "desktop workflows" ], "notes": "Counted unique identifiers in the inspected manifest. Normal and hard modes differ in how much task setup the agent must perform. Record the difficulty and observation/backend configuration with each result; no VM was launched.", "observations": [ "screenshots", "accessibility information", "screen-element detections" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "MIT", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/microsoft/WindowsAgentArena/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "e481abd422a4cc8b819ad84b32684e4af641e18a7c0d277dd608cfd5d3c5e6ad", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details" ], "url": "https://raw.githubusercontent.com/microsoft/WindowsAgentArena/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "e481abd422a4cc8b819ad84b32684e4af641e18a7c0d277dd608cfd5d3c5e6ad", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/microsoft/WindowsAgentArena/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:06:30Z", "role": "primary", "sha256": "c565dc02ab0049bf9710b87e2760dda8d7c92268ea78f0354952695ab59c0de3", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/microsoft/WindowsAgentArena/HEAD/docs/Develop-Tasks.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T02:06:30Z", "role": "primary", "sha256": "902fb24cf0a9e146bf6c29288e74a7254fe40294e49dd47363e0b15d65608fef", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/microsoft/WindowsAgentArena/HEAD/src/win-arena-container/client/evaluation_examples_windows/test_all.json" } ] }, { "id": "github:gair-nlp/windowsagentarena-v2", "name": "WindowsAgentArena-V2", "url": "https://github.com/GAIR-NLP/WindowsAgentArena-V2", "environments": [ "desktop", "windows" ], "profile": { "actions": [ "PyAutoGUI" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/README.md", "population": "full WindowsAgentArena-V2 suite reported by README", "qualifier": "exact", "unit": "tasks", "value": 141 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/README.md", "population": "full suite coverage reported by README", "qualifier": "exact", "unit": "Windows applications", "value": 11 }, { "basis": "computed", "evidence_url": "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/src/win-arena-container/client/evaluation_examples_windows/test_custom.json", "population": "shipped test_custom.json selection", "qualifier": "exact", "unit": "task IDs", "value": 20 } ], "evaluation": "Runner exposes four observation choices and uses PyAutoGUI. count.py sums the last score in each results.json and divides by the number of traj.jsonl files, multiplying by 100; it separately reports result/trajectory/error-file coverage. This denominator is not automatically the full benchmark size.", "evidence_urls": [ "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/README.md", "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/src/win-arena-container/client/run.py", "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/src/win-arena-container/client/evaluation_examples_windows/test_custom.json", "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/count.py", "https://api.github.com/repos/GAIR-NLP/WindowsAgentArena-V2/git/trees/main?recursive=1" ], "focus": [ "Windows desktop task execution", "revised task evaluators" ], "notes": "README sample config uses max_steps=30. The shipped selected-task index has 20 IDs, not 141; preserve the selected task list, trial ID, score coverage and VM snapshot when comparing runs. Docker/Windows-11 setup is required. No VM or benchmark was executed.", "observations": [ "screenshot", "accessibility tree", "screenshot plus accessibility tree", "set-of-marks" ], "setting": "interactive" }, "licenses": { "code": { "basis": "Reviewed repository license declaration for code; third-party dependencies, data, weights and executable environment images are separate.", "declared": "MIT", "retrieved_at": "2026-10-18T04:35:20Z", "url": "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:35:17Z", "role": "primary", "sha256": "1dd744ce565ec00b20ea5637eb7bc6112e97fb9044bd9a471f7cc4b7227ca344", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:35:16Z", "role": "primary", "sha256": "638ef9ee593a3a560c1d0b3fb2bc4a9d31a3e904c0111971924a6d66e1763aa6", "supports": [ "identity", "release_status", "details.repository_tree_sha", "details.benchmark_protocol" ], "url": "https://api.github.com/repos/GAIR-NLP/WindowsAgentArena-V2/git/trees/main?recursive=1" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:40:39Z", "role": "primary", "sha256": "494de9e74629ac2c55358629c0d9ba13703f36317057a3089388fbb6e80ca28f", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/src/win-arena-container/client/run.py" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:40:39Z", "role": "primary", "sha256": "e45ba7fdc7555f46ad1b4cfaab79d00754909d053fc49d5bab651b0fb593439b", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/src/win-arena-container/client/evaluation_examples_windows/test_custom.json" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-18T04:40:39Z", "role": "primary", "sha256": "f12147ac78c39c72eb75f9c37b293ba7fff3209fc5758fd51c69abbd96d80a97", "supports": [ "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/GAIR-NLP/WindowsAgentArena-V2/HEAD/count.py" } ] }, { "id": "github:hitsz-tmg/windowsworld", "name": "WindowsWorld", "url": "https://github.com/HITsz-TMG/WindowsWorld", "environments": [ "desktop", "windows" ], "profile": { "actions": [ "PyAutoGUI", "UiPath Computer_13 action interface" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/HITsz-TMG/WindowsWorld/HEAD/README.md", "population": "WindowsWorld benchmark described in the retained README", "qualifier": "exact", "unit": "tasks", "value": 181 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/HITsz-TMG/WindowsWorld/HEAD/README.md", "population": "WindowsWorld coverage described in the retained README", "qualifier": "exact", "unit": "desktop applications", "value": 17 } ], "evaluation": "Publisher distinguishes S_int (intermediate checkpoint score on L1–L3) and S_final (final completion on L1–L4). Documented step budgets are 15/25/40/20 for L1/L2/L3/L4. Preserve level, observation mode, agent interface and checkpoint/final metric when reporting a run.", "evidence_urls": [ "https://raw.githubusercontent.com/HITsz-TMG/WindowsWorld/HEAD/README.md", "https://api.github.com/repos/HITsz-TMG/WindowsWorld/git/trees/main?recursive=1" ], "focus": [ "multi-application Windows workflows", "intermediate-state and final-goal completion" ], "notes": "Interactive VMware/Windows environment. The public repository includes benchmark.json, task/evaluator code and monitoring support; the README refers VM import to Installation Guide.md, whose distribution links and payloads were not inspected for this profile. Checkpoint counts and application counts are not extra tasks. Apache-2.0 is recorded for repository code; Windows, apps and VM artifacts retain separate terms.", "observations": [ "screenshot", "accessibility tree", "screenshot plus accessibility tree", "set-of-marks" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:59Z", "url": "https://raw.githubusercontent.com/HITsz-TMG/WindowsWorld/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:59Z", "role": "primary", "sha256": "9857bd8ae9f521f152761ae3bd85debe4e7471aad34c9eb47618332948bca6c9", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol" ], "url": "https://raw.githubusercontent.com/HITsz-TMG/WindowsWorld/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T14:14:05Z", "role": "primary", "sha256": "490c9257d56f0a5214e6424e02490379b0b00ca2f7d4626fd5d14e17239d05ed", "supports": [ "details.benchmark_protocol" ], "url": "https://api.github.com/repos/HITsz-TMG/WindowsWorld/git/trees/main?recursive=1" } ] }, { "id": "github:hazyresearch/wonderbread", "name": "WONDERBREAD", "url": "https://github.com/HazyResearch/wonderbread", "environments": [ "web" ], "profile": { "actions": [ "generate procedures", "segment demonstrations", "answer workflow questions", "validate and rank procedures" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/HazyResearch/wonderbread/HEAD/README.md", "population": "WONDERBREAD BPM benchmark", "qualifier": "exact", "unit": "task types", "value": 6 }, { "basis": "reported", "evidence_url": "https://zenodo.org/api/records/14094162", "population": "full dataset release 14094162", "qualifier": "exact", "unit": "human demonstrations", "value": 2928 }, { "basis": "reported", "evidence_url": "https://zenodo.org/api/records/14094162", "population": "full dataset release 14094162", "qualifier": "exact", "unit": "web navigation workflows", "value": 598 }, { "basis": "reported", "evidence_url": "https://zenodo.org/api/records/14094162", "population": "gold subset used for the original paper evaluations", "qualifier": "exact", "unit": "human demonstrations", "value": 724 } ], "evaluation": "Task-specific scripts evaluate six workflow-understanding task types; model outputs are not treated as successful interactive execution.", "evidence_urls": [ "https://raw.githubusercontent.com/HazyResearch/wonderbread/HEAD/README.md", "https://zenodo.org/api/records/14094162" ], "focus": [ "workflow documentation", "workflow quality assessment", "workflow knowledge transfer" ], "notes": "Counts are publisher-reported. The gold subset covers 162 workflows and differs from the full collection; scores require the exact task and subset.", "observations": [ "screen recordings", "action traces", "screenshots", "written standard operating procedures" ], "setting": "offline" }, "licenses": { "code": null, "data": { "basis": "The linked Zenodo dataset release declares CC-BY-4.0; this does not assign a code license to the repository.", "declared": "cc-by-4.0", "retrieved_at": "2026-10-17T09:15:34Z", "url": "https://zenodo.org/api/records/14094162" }, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T09:12:11Z", "role": "primary", "sha256": "1e4308d59a22b39543c28b745a58d3d7c47be1ab9df9a4c5c0fe868a810350b2", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "notes", "relations", "details" ], "url": "https://raw.githubusercontent.com/HazyResearch/wonderbread/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-17T09:15:34Z", "role": "primary", "sha256": "68b4b952c1ab90484d457b392de1d0c95a24f6cc1674c82125a6bd26f1cb1016", "supports": [ "release_status", "licenses.data", "details.benchmark_protocol" ], "url": "https://zenodo.org/api/records/14094162" } ] }, { "id": "github:servicenow/workarena", "name": "WorkArena", "url": "https://github.com/ServiceNow/WorkArena", "environments": [ "web" ], "profile": { "actions": [ "BrowserGym browser actions" ], "counts": [ { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ServiceNow/WorkArena/HEAD/README.md", "population": "WorkArena-L1 atomic tasks", "qualifier": "exact", "unit": "task definitions", "value": 33 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ServiceNow/WorkArena/HEAD/README.md", "population": "WorkArena-L1 documented instances", "qualifier": "exact", "unit": "task instances", "value": 19912 }, { "basis": "reported", "evidence_url": "https://raw.githubusercontent.com/ServiceNow/WorkArena/HEAD/README.md", "population": "WorkArena++ compositional tasks", "qualifier": "exact", "unit": "task definitions", "value": 682 } ], "evaluation": "Task-specific checks on ServiceNow workflows; levels and task configurations define different evaluations.", "evidence_urls": [ "https://raw.githubusercontent.com/ServiceNow/WorkArena/HEAD/README.md", "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=800&max_results=100&sortBy=submittedDate&sortOrder=descending" ], "focus": [ "enterprise web workflows", "compositional tasks" ], "notes": "Definitions and instantiated tasks are different units. ServiceNow instance access is gated. WorkArena-L1 and WorkArena++ should be reported separately.", "observations": [ "multimodal browser observations" ], "setting": "interactive" }, "licenses": { "code": { "basis": "repository LICENSE header; not a weights/data license", "declared": "Apache-2.0", "retrieved_at": "2026-10-16T06:41:57Z", "url": "https://raw.githubusercontent.com/ServiceNow/WorkArena/HEAD/LICENSE" }, "data": null, "document": null, "weights": null }, "evidence": [ { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:35:57Z", "role": "primary", "sha256": "834a48f0346036a0fc591a36f9eb6f513b44a2777a66afa0917f28fc63f29334", "supports": [ "name", "kind", "summary", "environments", "topics", "scope", "relations", "details.benchmark_protocol", "details.leaderboard_url" ], "url": "https://raw.githubusercontent.com/ServiceNow/WorkArena/HEAD/README.md" }, { "method": "editorial_primary_review", "retrieved_at": "2026-10-16T06:36:58Z", "role": "primary", "sha256": "c87e97eb397da04806d53145cf9c13be739fc657bb54dca9318c8c4ed2a36249", "supports": [ "details.benchmark_protocol" ], "url": "https://export.arxiv.org/api/query?search_query=%28ti%3A%22computer+use%22+OR+ti%3A%22computer-use%22+OR+ti%3A%22GUI+agent%22+OR+ti%3A%22GUI+agents%22+OR+ti%3A%22GUI+grounding%22+OR+ti%3A%22web+agents%22+OR+ti%3A%22browser+agent%22%29+AND+submittedDate%3A%5B200001010000+TO+202610160636%5D&start=800&max_results=100&sortBy=submittedDate&sortOrder=descending" } ] } ] }