{
  "name": "NewLab Work Agent Reality Index",
  "version": "0.1-draft",
  "published_at": "2026-10-09",
  "status": "protocol-design-only",
  "results": [],
  "notice": "No agent has been tested against this candidate suite. Tasks are specifications, not frozen fixtures or benchmark scores.",
  "methodology_url": "https://newlab.si/en/benchmarks/work-agent-reality-index/",
  "rules": {
    "task_repeats": 3,
    "fresh_context_each_repeat": true,
    "human_interventions": "Count and disclose; no unreported assistance",
    "success": "Task-specific acceptance gate with artifact inspection",
    "cost": "Record actual paid API cost; unavailable is null, never zero",
    "latency": "Wall-clock from task handoff to submitted artifact",
    "safety": "Disallow live account mutations, secrets, destructive actions and unauthorized access",
    "comparability": "Freeze fixtures, time window, tool permissions, budgets, harness and versions before comparisons"
  },
  "tasks": [
    {
      "id": "REP-01",
      "family": "repository",
      "title": "Repair a failing regression test",
      "goal": "Given a small isolated TypeScript repository with one reproducible failing test, find and fix the root cause without weakening the assertion.",
      "pass": "Clean install and the unchanged hidden regression suite pass; patch touches only allowed files.",
      "validator": "Automated test suite, diff allowlist, and hidden-case checks.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "REP-02",
      "family": "repository",
      "title": "Refactor configuration without behavior drift",
      "goal": "Move duplicated configuration into a typed module in a synthetic repository while preserving runtime outputs.",
      "pass": "Snapshot outputs match the reference and typecheck passes; no secrets or dependencies added.",
      "validator": "Snapshot comparison, typecheck and dependency diff.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "REP-03",
      "family": "repository",
      "title": "Fix documentation/CLI drift",
      "goal": "Correct an intentionally outdated README command against a sandbox CLI and add a smoke check.",
      "pass": "Documented command executes successfully from a clean checkout and smoke check passes.",
      "validator": "Fresh-environment command replay and file diff.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "WEB-01",
      "family": "browser",
      "title": "Resolve conflicting public documentation",
      "goal": "Find two contradictory versions of a public API claim and establish which version applies to a frozen target release.",
      "pass": "Provide primary-source URLs, exact version context and a correct conclusion without invented citations.",
      "validator": "Source-link audit and version-grounded human review.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "WEB-02",
      "family": "browser",
      "title": "Reconcile a dated feature comparison",
      "goal": "Compare two public tools against a fixed requirement list using vendor documentation captured on a specified date.",
      "pass": "Every comparison cell is traceable to a URL or marked unknown; no fabricated pricing.",
      "validator": "Source audit and completeness rubric.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "WEB-03",
      "family": "browser",
      "title": "Navigate a documentation migration",
      "goal": "Locate replacement documentation for a deprecated API and produce a migration checklist without touching live accounts.",
      "pass": "Checklist links to current official docs and correctly identifies breaking changes.",
      "validator": "Link validity and manually reviewed change list.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "ART-01",
      "family": "artifact",
      "title": "Research to auditable CSV",
      "goal": "Extract a small frozen set of public product specifications into a CSV with a per-row source URL.",
      "pass": "CSV schema, row count, numeric types and source provenance pass deterministic validation.",
      "validator": "CSV schema, frozen answer key and citation audit.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "ART-02",
      "family": "artifact",
      "title": "Conflicting evidence to concise memo",
      "goal": "Write a brief from a supplied bundle of partially conflicting public documents, distinguishing observation from inference.",
      "pass": "Required sections present; key facts grounded; disagreements explicitly noted.",
      "validator": "Document structure and evidence rubric with blind review.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "ART-03",
      "family": "artifact",
      "title": "Generate a formula-correct workbook",
      "goal": "Turn a synthetic monthly operations dataset into a spreadsheet with formulas and a short interpretation.",
      "pass": "Workbook opens, required formulas recalculate to expected values and no rows are dropped.",
      "validator": "OpenXML inspection and reference calculation.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "CROSS-01",
      "family": "cross-interface",
      "title": "Release-readiness audit without deployment",
      "goal": "Inspect an isolated repo, its build output and public staging URL; prepare a release checklist without making a deployment.",
      "pass": "Report identifies seeded blockers, cites commands and HTTP evidence, and does not mutate production.",
      "validator": "Seeded-issue answer key, command log and side-effect audit.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "CROSS-02",
      "family": "cross-interface",
      "title": "Reconcile synthetic records across files and UI",
      "goal": "Compare an offline synthetic CSV with a read-only local web interface and produce a discrepancy report.",
      "pass": "All seeded discrepancies found with row IDs; original data unchanged.",
      "validator": "Deterministic discrepancy key and file integrity hash.",
      "status": "candidate-not-frozen",
      "results": []
    },
    {
      "id": "CROSS-03",
      "family": "cross-interface",
      "title": "Resume interrupted multi-stage work",
      "goal": "Complete a sandbox research-to-report job from a partial checkpoint, avoiding duplicate steps and preserving prior evidence.",
      "pass": "Final artifact complete, no duplicated work, state transitions documented and all original evidence retained.",
      "validator": "Checkpoint/event-log replay and final artifact rubric.",
      "status": "candidate-not-frozen",
      "results": []
    }
  ]
}