{ "actor": { "model": "claude-sonnet-5", "profile_id": "claude-code/claude-sonnet-5/controlled-agent-task-v1", "provider": "claude-code", "response_parser": "claude-json" }, "agent_config_fingerprint": "12598b20398b5a673eb3aa04ad20d4f74df7e96726073c47617a095448135264", "attempted_runs": 15, "claim_boundary": "Controlled agent task performance; not an AI-readiness certification.", "complete": true, "completed_runs": 15, "evidence_lock_fingerprint": "77c7c9f588b1cb64f68cc7cf0cfe3a69e1e0229c29a6748ec499a5bcda7ddb72", "expected_runs": 15, "generated_at_epoch": 1787751037, "metrics": { "cache_creation_input_tokens": 626530, "cache_read_input_tokens": 193976, "citation_locator_validity": 1.0, "citation_normalizations": 0, "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": 1.0, "harness_outcomes": { "accessibility": { "failed": 0, "not-run": 0, "passed": 6 }, "governance-process": { "failed": 0, "not-run": 0, "passed": 3 }, "interaction": { "failed": 2, "not-run": 0, "passed": 4 }, "render": { "failed": 0, "not-run": 0, "passed": 3 }, "responsive": { "failed": 0, "not-run": 0, "passed": 6 }, "semantic-selection": { "failed": 0, "not-run": 0, "passed": 3 }, "typecheck": { "failed": 0, "not-run": 0, "passed": 9 } }, "human_interventions": 0, "input_tokens": 820527, "invalid_component_usages": 0, "invalid_prop_usages": 0, "invalid_token_usages": 0, "mean_actor_duration_seconds": 61.16, "mean_duration_seconds": 62.69, "mean_exact_answer_reproducibility": 0.4666, "mean_harness_duration_seconds": 1.53, "mean_outcome_agreement": 0.9333, "output_tokens": 98017, "required_provenance_coverage": 1.0, "task_completion_rate": 1.0, "task_success_rate": 0.8667, "total_cost_usd": 5.7915808, "uncached_input_tokens": 21 }, "publication_issues": [], "publication_ready": true, "report_fingerprint": "c82efc9ce853e74082337d5ec77f11df7886295134c33512cad6bbc94dc0d1f4", "report_format": "ds-eval/agent-performance-report@1", "stale_runs": 0, "suite": "design-system-agent-tasks/1.0.0", "system": "carbon-core-v11", "tasks": [ { "area": "foundations", "attempted_runs": 3, "completed_runs": 3, "exact_answer_reproducibility": 1.0, "expected_runs": 3, "outcome_agreement": 1.0, "runs": [ { "actor_duration_ms": 26486, "duration_ms": 26537, "error": null, "harness_duration_ms": 51, "human_interventions": 0, "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 36861, "cache_read_input_tokens": 43892, "cost_usd": 0.29774459999999997, "input_tokens": 80756, "output_tokens": 2257, "uncached_input_tokens": 3 }, "validation": { "answer_fingerprint": "93e2e332d66351c89e27c1fd221a37550bd5e77b5b4e929cf9aa3d73ef11cd48", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": 1.0, "harness_outcomes": { "semantic-selection": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "line 7643", "path": "figma-export.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 7636-7649", "path": "figma-export.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "line 237", "path": "files/packages/themes/src/tokens/v11TokenGroup.ts", "snapshot_id": "5143be78329c1b1e4237" }, { "locator": "lines 233-256", "path": "files/packages/themes/src/tokens/v11TokenGroup.ts", "snapshot_id": "5143be78329c1b1e4237" } ] } }, { "actor_duration_ms": 24231, "duration_ms": 24295, "error": null, "harness_duration_ms": 64, "human_interventions": 0, "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 35364, "cache_read_input_tokens": 86060, "cost_usd": 0.298719, "input_tokens": 121429, "output_tokens": 2077, "uncached_input_tokens": 5 }, "validation": { "answer_fingerprint": "93e2e332d66351c89e27c1fd221a37550bd5e77b5b4e929cf9aa3d73ef11cd48", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": 1.0, "harness_outcomes": { "semantic-selection": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 7638-7649", "path": "figma-export.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 233-256", "path": "files/packages/themes/src/tokens/v11TokenGroup.ts", "snapshot_id": "5143be78329c1b1e4237" } ] } }, { "actor_duration_ms": 10135, "duration_ms": 10195, "error": null, "harness_duration_ms": 60, "human_interventions": 0, "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 35504, "cache_read_input_tokens": 4194, "cost_usd": 0.2552322, "input_tokens": 39699, "output_tokens": 760, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "93e2e332d66351c89e27c1fd221a37550bd5e77b5b4e929cf9aa3d73ef11cd48", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": 1.0, "harness_outcomes": { "semantic-selection": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "line 7643", "path": "figma-export.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "line 237", "path": "files/packages/themes/src/tokens/v11TokenGroup.ts", "snapshot_id": "5143be78329c1b1e4237" }, { "locator": "lines 233-256", "path": "files/packages/themes/src/tokens/v11TokenGroup.ts", "snapshot_id": "5143be78329c1b1e4237" } ] } } ], "stale_runs": 0, "successful_runs": 3, "task_completion_rate": 1.0, "task_id": "carbon-semantic-text-token", "task_success_rate": 1.0, "type": "semantic-token-selection" }, { "area": "bindings", "attempted_runs": 3, "completed_runs": 3, "exact_answer_reproducibility": 0.3333, "expected_runs": 3, "outcome_agreement": 1.0, "runs": [ { "actor_duration_ms": 19905, "duration_ms": 21817, "error": null, "harness_duration_ms": 1912, "human_interventions": 0, "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 41636, "cache_read_input_tokens": 4194, "cost_usd": 0.31772219999999995, "input_tokens": 45831, "output_tokens": 2048, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "b09ecc1526c1c60d8db7761424e853b380e353b06bba8e8a746af82c6b8752d0", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 86-98", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 61-73", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 74-85", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 49-52", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 145-148", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 13-51", "path": "files/packages/react/code-connect/Button/Button.figma.tsx", "snapshot_id": "ef0fc94e689c12c30608" } ] } }, { "actor_duration_ms": 16735, "duration_ms": 18312, "error": null, "harness_duration_ms": 1577, "human_interventions": 0, "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 40055, "cache_read_input_tokens": 5775, "cost_usd": 0.3020405, "input_tokens": 45831, "output_tokens": 1603, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "25f38b925097cb86f9cd398fbb833c1d429a05fbcbe228963340e02905114a36", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 61-73", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 74-85", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 86-98", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 13-51", "path": "files/packages/react/code-connect/Button/Button.figma.tsx", "snapshot_id": "ef0fc94e689c12c30608" } ] } }, { "actor_duration_ms": 17623, "duration_ms": 19181, "error": null, "harness_duration_ms": 1558, "human_interventions": 0, "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 41636, "cache_read_input_tokens": 4194, "cost_usd": 0.3145172, "input_tokens": 45831, "output_tokens": 1834, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "37bb2fd4c9409dbdf0ace83f5d341fcbf21c19ac5632d6d659ee60d862819ad7", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 86-98", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 61-73", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 74-85", "path": "figma-sample-details.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 5-20", "path": "figma-sample-resolution.json", "snapshot_id": "31e36527e62be3c4f265" }, { "locator": "lines 17-38", "path": "files/packages/react/code-connect/Button/Button.figma.tsx", "snapshot_id": "ef0fc94e689c12c30608" }, { "locator": "lines 47-49", "path": "files/packages/react/code-connect/Button/Button.figma.tsx", "snapshot_id": "ef0fc94e689c12c30608" } ] } } ], "stale_runs": 0, "successful_runs": 3, "task_completion_rate": 1.0, "task_id": "carbon-button-design-to-code", "task_success_rate": 1.0, "type": "design-to-code-binding" }, { "area": "components", "attempted_runs": 3, "completed_runs": 3, "exact_answer_reproducibility": 0.3333, "expected_runs": 3, "outcome_agreement": 1.0, "runs": [ { "actor_duration_ms": 109486, "duration_ms": 111399, "error": null, "harness_duration_ms": 1913, "human_interventions": 0, "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 29074, "cache_read_input_tokens": 4193, "cost_usd": 0.37041189999999996, "input_tokens": 33268, "output_tokens": 11523, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "d8ac665214b46bf2e096b7e0b94e343abde0300731dea6a7fcd4c2019f0f320b", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 354-379", "path": "content.txt", "snapshot_id": "ddbd1f2a7617a60a1cdc" }, { "locator": "lines 73-97", "path": "files/packages/react/src/components/TextInput/TextInput.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 232-257", "path": "files/packages/react/src/components/TextInput/TextInput.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 15-23", "path": "files/packages/react/src/components/Form/Form.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 424-427", "path": "files/packages/react/src/components/Button/Button.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 41-61", "path": "files/e2e/components/TextInput/TextInput-test.avt.e2e.js", "snapshot_id": "c7035884c47c4a634eff" }, { "locator": "lines 118-137", "path": "files/docs/guides/accessibility.md", "snapshot_id": "c7035884c47c4a634eff" } ] } }, { "actor_duration_ms": 84885, "duration_ms": 86805, "error": null, "harness_duration_ms": 1920, "human_interventions": 0, "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 27494, "cache_read_input_tokens": 5773, "cost_usd": 0.3239359, "input_tokens": 33268, "output_tokens": 9025, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "e1db41ecc342b8b300f0f557893263c8bfd94b3d70dc88fd72af467fd062acde", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 354-379", "path": "content.txt", "snapshot_id": "ddbd1f2a7617a60a1cdc" }, { "locator": "lines 74-91", "path": "files/packages/react/src/components/TextInput/TextInput.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 232-257", "path": "files/packages/react/src/components/TextInput/TextInput.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 15-23", "path": "files/packages/react/src/components/Form/Form.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 424-428", "path": "files/packages/react/src/components/Button/Button.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 49-54", "path": "files/e2e/components/TextInput/TextInput-test.avt.e2e.js", "snapshot_id": "c7035884c47c4a634eff" } ] } }, { "actor_duration_ms": 96687, "duration_ms": 98609, "error": null, "harness_duration_ms": 1922, "human_interventions": 0, "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 29074, "cache_read_input_tokens": 4193, "cost_usd": 0.35187189999999996, "input_tokens": 33268, "output_tokens": 10287, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "265ab5be7f512adcf12709206736c958f82a923fb958ce0bba21b5c080225772", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 354-379", "path": "content.txt", "snapshot_id": "ddbd1f2a7617a60a1cdc" }, { "locator": "lines 15-23", "path": "files/packages/react/src/components/Form/Form.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 232-257", "path": "files/packages/react/src/components/TextInput/TextInput.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 146-172", "path": "files/packages/react/src/components/Button/Button.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 25-39", "path": "files/e2e/components/TextInput/TextInput-test.avt.e2e.js", "snapshot_id": "c7035884c47c4a634eff" }, { "locator": "lines 41-61", "path": "files/e2e/components/TextInput/TextInput-test.avt.e2e.js", "snapshot_id": "c7035884c47c4a634eff" } ] } } ], "stale_runs": 0, "successful_runs": 3, "task_completion_rate": 1.0, "task_id": "carbon-resource-name-field", "task_success_rate": 1.0, "type": "component-behavior-implementation" }, { "area": "structure", "attempted_runs": 3, "completed_runs": 3, "exact_answer_reproducibility": 0.3333, "expected_runs": 3, "outcome_agreement": 0.6667, "runs": [ { "actor_duration_ms": 176577, "duration_ms": 180236, "error": null, "harness_duration_ms": 3659, "human_interventions": 0, "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 73666, "cache_read_input_tokens": 5779, "cost_usd": 0.7890317, "input_tokens": 79446, "output_tokens": 19153, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "08d9664f8219fc9418e47feb6c055f27e39e2d48f5dd642d1bea3d7808beee78", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "failed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [ "interaction" ], "missing_required_props": [], "passed": false, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 147-175", "path": "content.txt", "snapshot_id": "864cd765d8e0c3ebb4ae" }, { "locator": "lines 296-299", "path": "content.txt", "snapshot_id": "3eb605429f1ccd49de1b" }, { "locator": "lines 524-528", "path": "content.txt", "snapshot_id": "3eb605429f1ccd49de1b" }, { "locator": "lines 248-249", "path": "content.txt", "snapshot_id": "3eb605429f1ccd49de1b" }, { "locator": "lines 119-126", "path": "content.txt", "snapshot_id": "30a9f6ff2e51428b61e6" }, { "locator": "lines 208-215", "path": "content.txt", "snapshot_id": "ccdd6def0597b441795c" }, { "locator": "lines 460-468", "path": "content.txt", "snapshot_id": "ccdd6def0597b441795c" }, { "locator": "lines 15-21", "path": "files/packages/react/src/components/Grid/Grid.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 44-73", "path": "files/packages/react/src/components/Grid/Column.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 15-23", "path": "files/packages/react/src/components/Form/Form.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 74-97", "path": "files/packages/react/src/components/TextInput/TextInput.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 232-257", "path": "files/packages/react/src/components/TextInput/TextInput.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 79-99", "path": "files/packages/react/src/components/Toggle/Toggle.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 146-151", "path": "files/packages/react/src/components/Button/Button.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 26-44", "path": "files/packages/react/src/components/Loading/Loading.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 619-664", "path": "files/packages/react/src/components/Notification/Notification.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" } ] } }, { "actor_duration_ms": 135860, "duration_ms": 140348, "error": null, "harness_duration_ms": 4488, "human_interventions": 0, "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 73666, "cache_read_input_tokens": 5779, "cost_usd": 0.7244217, "input_tokens": 79446, "output_tokens": 14845, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "7c703f02594e5aab41715b6df8969d0b69b7166fff83ae0ac44b40f3e983b9b6", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 147-175", "path": "content.txt", "snapshot_id": "864cd765d8e0c3ebb4ae" }, { "locator": "lines 523-528", "path": "content.txt", "snapshot_id": "3eb605429f1ccd49de1b" }, { "locator": "lines 123-126", "path": "content.txt", "snapshot_id": "30a9f6ff2e51428b61e6" }, { "locator": "lines 454-468", "path": "content.txt", "snapshot_id": "ccdd6def0597b441795c" }, { "locator": "lines 51-53", "path": "files/packages/react/src/components/Grid/Grid.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 65-74", "path": "files/packages/react/src/components/Grid/Column.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 15-23", "path": "files/packages/react/src/components/Form/Form.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 232-257", "path": "files/packages/react/src/components/TextInput/TextInput.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 92-99", "path": "files/packages/react/src/components/Toggle/Toggle.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 146-170", "path": "files/packages/react/src/components/Button/Button.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 46-53", "path": "files/packages/react/src/components/Loading/Loading.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 666-680", "path": "files/packages/react/src/components/Notification/Notification.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" } ] } }, { "actor_duration_ms": 91342, "duration_ms": 95012, "error": null, "harness_duration_ms": 3670, "human_interventions": 0, "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 73666, "cache_read_input_tokens": 5779, "cost_usd": 0.6605466999999999, "input_tokens": 79446, "output_tokens": 10587, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "424f17cccf1a6dffbee51366c1b2d503189c14146fd99abeb8e61ad5bcb9600c", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "failed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [ "interaction" ], "missing_required_props": [], "passed": false, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 147-175", "path": "content.txt", "snapshot_id": "864cd765d8e0c3ebb4ae" }, { "locator": "lines 244-249", "path": "content.txt", "snapshot_id": "3eb605429f1ccd49de1b" }, { "locator": "lines 517-528", "path": "content.txt", "snapshot_id": "3eb605429f1ccd49de1b" }, { "locator": "lines 110-116", "path": "content.txt", "snapshot_id": "30a9f6ff2e51428b61e6" }, { "locator": "lines 125-126", "path": "content.txt", "snapshot_id": "30a9f6ff2e51428b61e6" }, { "locator": "lines 208-215", "path": "content.txt", "snapshot_id": "ccdd6def0597b441795c" }, { "locator": "lines 221-231", "path": "content.txt", "snapshot_id": "ccdd6def0597b441795c" }, { "locator": "lines 50-53", "path": "files/packages/react/src/components/Grid/Grid.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 15-23", "path": "files/packages/react/src/components/Form/Form.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 252-256", "path": "files/packages/react/src/components/TextInput/TextInput.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 94-134", "path": "files/packages/react/src/components/Toggle/Toggle.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 146-170", "path": "files/packages/react/src/components/Button/Button.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 26-53", "path": "files/packages/react/src/components/Loading/Loading.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" }, { "locator": "lines 666-744", "path": "files/packages/react/src/components/Notification/Notification.tsx", "snapshot_id": "8ecdf97a7dd31b0f1380" } ] } } ], "stale_runs": 0, "successful_runs": 1, "task_completion_rate": 1.0, "task_id": "carbon-create-configure-save-flow", "task_success_rate": 0.3333, "type": "flow-composition" }, { "area": "governance", "attempted_runs": 3, "completed_runs": 3, "exact_answer_reproducibility": 0.3333, "expected_runs": 3, "outcome_agreement": 1.0, "runs": [ { "actor_duration_ms": 33410, "duration_ms": 33463, "error": null, "harness_duration_ms": 53, "human_interventions": 0, "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 30139, "cache_read_input_tokens": 4196, "cost_usd": 0.2616528, "input_tokens": 34336, "output_tokens": 3796, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "2b2d7bdfa32f108d73cd84bc88ca8a9b088094ffcab0826a6eade58ef99596ad", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "governance-process": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 57-71", "path": "files/docs/guides/versioning.md", "snapshot_id": "3eb54126648a690950d8" }, { "locator": "lines 111-130", "path": "files/docs/guides/versioning.md", "snapshot_id": "3eb54126648a690950d8" }, { "locator": "lines 132-153", "path": "files/docs/guides/versioning.md", "snapshot_id": "3eb54126648a690950d8" }, { "locator": "lines 292-299", "path": "files/docs/guides/versioning.md", "snapshot_id": "3eb54126648a690950d8" }, { "locator": "lines 45-48", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 50-59", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 67-75", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 84-93", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 102-104", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 163-183", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "line 1", "path": "files/.github/PULL_REQUEST_TEMPLATE.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 5-21", "path": "files/.github/PULL_REQUEST_TEMPLATE.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 31-42", "path": "files/.github/PULL_REQUEST_TEMPLATE.md", "snapshot_id": "850be183e89b93a74a5d" } ] } }, { "actor_duration_ms": 38120, "duration_ms": 38168, "error": null, "harness_duration_ms": 48, "human_interventions": 0, "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 28556, "cache_read_input_tokens": 5779, "cost_usd": 0.25867470000000004, "input_tokens": 34336, "output_tokens": 4199, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "877433657faa8275631a9b9996fe01ce66e8bade24b999933bcb1c8f2bb9f09a", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "governance-process": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 57-71", "path": "files/docs/guides/versioning.md", "snapshot_id": "3eb54126648a690950d8" }, { "locator": "lines 111-130", "path": "files/docs/guides/versioning.md", "snapshot_id": "3eb54126648a690950d8" }, { "locator": "lines 132-153", "path": "files/docs/guides/versioning.md", "snapshot_id": "3eb54126648a690950d8" }, { "locator": "lines 39-49", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 50-61", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 63-75", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 84-93", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 102-104", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 163-183", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 1-21", "path": "files/.github/PULL_REQUEST_TEMPLATE.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 33-42", "path": "files/.github/PULL_REQUEST_TEMPLATE.md", "snapshot_id": "850be183e89b93a74a5d" } ] } }, { "actor_duration_ms": 35967, "duration_ms": 36019, "error": null, "harness_duration_ms": 52, "human_interventions": 0, "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 30139, "cache_read_input_tokens": 4196, "cost_usd": 0.2650578, "input_tokens": 34336, "output_tokens": 4023, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "6f63229b574079e91e53c335be6ac2ab10a7f28893b21a355501800fdb76f570", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "governance-process": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "selected_token_normalizations": [], "task_completed": true, "valid_citations": [ { "locator": "lines 45-48", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "line 1", "path": "files/.github/PULL_REQUEST_TEMPLATE.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 57-71", "path": "files/docs/guides/versioning.md", "snapshot_id": "3eb54126648a690950d8" }, { "locator": "lines 111-130", "path": "files/docs/guides/versioning.md", "snapshot_id": "3eb54126648a690950d8" }, { "locator": "lines 132-153", "path": "files/docs/guides/versioning.md", "snapshot_id": "3eb54126648a690950d8" }, { "locator": "lines 163-183", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 19-21", "path": "files/.github/PULL_REQUEST_TEMPLATE.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 35-38", "path": "files/.github/PULL_REQUEST_TEMPLATE.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 67-75", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 84-93", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" }, { "locator": "lines 102-104", "path": "files/docs/guides/reviewing-pull-requests.md", "snapshot_id": "850be183e89b93a74a5d" } ] } } ], "stale_runs": 0, "successful_runs": 3, "task_completion_rate": 1.0, "task_id": "carbon-breaking-prop-governance", "task_success_rate": 1.0, "type": "governed-change-proposal" } ] }