{ "actor": { "model": "claude-sonnet-5", "profile_id": "claude-code/claude-sonnet-5/controlled-agent-task-v1", "provider": "claude-code", "response_parser": "claude-json" }, "agent_config_fingerprint": "5cdc6591a83b560e99b334fb7bd89020fc1fc49401b6a4b90cc0823c4135c140", "attempted_runs": 15, "claim_boundary": "Controlled agent task performance; not an AI-readiness certification.", "complete": true, "completed_runs": 15, "evidence_lock_fingerprint": "742626b9d8bf76fade5adc17a10bf0aacf67a9f7d8ec7304b92d3d71eb009d7f", "expected_runs": 15, "generated_at_epoch": 1787494269, "metrics": { "cache_creation_input_tokens": 662511, "cache_read_input_tokens": 71210, "citation_locator_validity": 1.0, "citation_normalizations": 0, "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": 1.0, "harness_outcomes": { "accessibility": { "failed": 1, "not-run": 0, "passed": 5 }, "governance-process": { "failed": 0, "not-run": 0, "passed": 3 }, "interaction": { "failed": 1, "not-run": 0, "passed": 5 }, "render": { "failed": 0, "not-run": 0, "passed": 3 }, "responsive": { "failed": 0, "not-run": 0, "passed": 6 }, "semantic-selection": { "failed": 0, "not-run": 0, "passed": 3 }, "typecheck": { "failed": 0, "not-run": 0, "passed": 9 } }, "human_interventions": 0, "input_tokens": 733740, "invalid_component_usages": 0, "invalid_prop_usages": 0, "invalid_token_usages": 0, "mean_actor_duration_seconds": 63.85, "mean_duration_seconds": 67.99, "mean_exact_answer_reproducibility": 0.4666, "mean_harness_duration_seconds": 4.14, "mean_outcome_agreement": 0.9333, "output_tokens": 105553, "required_provenance_coverage": 1.0, "task_completion_rate": 1.0, "task_success_rate": 0.9333, "total_cost_usd": 6.069603999999999, "uncached_input_tokens": 19 }, "publication_issues": [], "publication_ready": true, "report_fingerprint": "2436e4e718b94e92a916b0085a1a68d05bc3e2166bcbcaf0fb145e88968e8a71", "report_format": "ds-eval/agent-performance-report@1", "stale_runs": 0, "suite": "design-system-agent-tasks/1.0.0", "system": "primer", "tasks": [ { "area": "foundations", "attempted_runs": 3, "completed_runs": 3, "exact_answer_reproducibility": 1.0, "expected_runs": 3, "outcome_agreement": 1.0, "runs": [ { "actor_duration_ms": 9524, "duration_ms": 9583, "error": null, "harness_duration_ms": 59, "human_interventions": 0, "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 21492, "cache_read_input_tokens": 21057, "cost_usd": 0.1597831, "input_tokens": 42552, "output_tokens": 750, "uncached_input_tokens": 3 }, "validation": { "answer_fingerprint": "af050c9ee7e8c5f9a1fc4b98607a8c4bb95ff78e1dbd36bf6a97ef2b8ff10e89", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": 1.0, "harness_outcomes": { "semantic-selection": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 15935-15950", "path": "figma-export.json", "snapshot_id": "dfc83d5c91c7186f6315" } ] } }, { "actor_duration_ms": 9847, "duration_ms": 9900, "error": null, "harness_duration_ms": 53, "human_interventions": 0, "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 20791, "cache_read_input_tokens": 0, "cost_usd": 0.14931399999999997, "input_tokens": 20792, "output_tokens": 754, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "af050c9ee7e8c5f9a1fc4b98607a8c4bb95ff78e1dbd36bf6a97ef2b8ff10e89", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": 1.0, "harness_outcomes": { "semantic-selection": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 15935-15944", "path": "figma-export.json", "snapshot_id": "dfc83d5c91c7186f6315" } ] } }, { "actor_duration_ms": 18404, "duration_ms": 18459, "error": null, "harness_duration_ms": 55, "human_interventions": 0, "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 17353, "cache_read_input_tokens": 24986, "cost_usd": 0.15005280000000001, "input_tokens": 42342, "output_tokens": 1678, "uncached_input_tokens": 3 }, "validation": { "answer_fingerprint": "af050c9ee7e8c5f9a1fc4b98607a8c4bb95ff78e1dbd36bf6a97ef2b8ff10e89", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": 1.0, "harness_outcomes": { "semantic-selection": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 15935-15945", "path": "figma-export.json", "snapshot_id": "dfc83d5c91c7186f6315" } ] } } ], "stale_runs": 0, "successful_runs": 3, "task_completion_rate": 1.0, "task_id": "primer-semantic-foreground-token", "task_success_rate": 1.0, "type": "semantic-token-selection" }, { "area": "bindings", "attempted_runs": 3, "completed_runs": 3, "exact_answer_reproducibility": 0.3333, "expected_runs": 3, "outcome_agreement": 1.0, "runs": [ { "actor_duration_ms": 64773, "duration_ms": 67354, "error": null, "harness_duration_ms": 2581, "human_interventions": 0, "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 36130, "cache_read_input_tokens": 0, "cost_usd": 0.35932600000000003, "input_tokens": 36131, "output_tokens": 7684, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "2cb8f101439c05bb8ff84931dd94afa23c70d48696e9c8862941baacf2058a75", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 1278-1287", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1235-1243", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1244-1255", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1231-1234", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1274-1277", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 17-22", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "line 6", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "lines 8-12", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "line 5", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "lines 38-49", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" } ] } }, { "actor_duration_ms": 34994, "duration_ms": 37697, "error": null, "harness_duration_ms": 2703, "human_interventions": 0, "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 35864, "cache_read_input_tokens": 0, "cost_usd": 0.30196, "input_tokens": 35865, "output_tokens": 3966, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "cd7a6751aa6e36df153532fc4def763dbd972faebd81ca3f5f7ea741cc47a402", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 1314-1320", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1278-1287", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1235-1243", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1244-1255", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1231-1234", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1274-1277", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 17-22", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "line 6", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "lines 8-12", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "lines 23-30", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "lines 38-49", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" } ] } }, { "actor_duration_ms": 31697, "duration_ms": 33869, "error": null, "harness_duration_ms": 2172, "human_interventions": 0, "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 31671, "cache_read_input_tokens": 4193, "cost_usd": 0.2719849, "input_tokens": 35865, "output_tokens": 3561, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "1a8763bf21a27f2053f4abac8e354c7d4b1e528031176601e55fac843aa1a6dc", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 1278-1287", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1244-1255", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1235-1243", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1231-1234", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 1274-1277", "path": "figma-sample-details.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 3-65", "path": "figma-sample-resolution.json", "snapshot_id": "dfc83d5c91c7186f6315" }, { "locator": "lines 17-22", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "line 6", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "lines 8-12", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "line 5", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "lines 23-30", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" }, { "locator": "lines 33-51", "path": "files/packages/react/src/Button/Button.figma.tsx", "snapshot_id": "172db810d1a67c7273e2" } ] } } ], "stale_runs": 0, "successful_runs": 3, "task_completion_rate": 1.0, "task_id": "primer-button-design-to-code", "task_success_rate": 1.0, "type": "design-to-code-binding" }, { "area": "components", "attempted_runs": 3, "completed_runs": 3, "exact_answer_reproducibility": 0.3333, "expected_runs": 3, "outcome_agreement": 0.6667, "runs": [ { "actor_duration_ms": 64208, "duration_ms": 67202, "error": null, "harness_duration_ms": 2994, "human_interventions": 0, "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 55877, "cache_read_input_tokens": 0, "cost_usd": 0.48245, "input_tokens": 55878, "output_tokens": 7201, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "94af5e0e3ff8cb7680a1f5c58599e4a46f74ba1ddaf6731ed377ed22a8bd298b", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 305-315", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 334-349", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 149-160", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 460-473", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 46-49", "path": "files/packages/react/src/TextInput/TextInput.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 543-551", "path": "files/packages/react/src/TextInput/TextInput.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 621-622", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "lines 535-544", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "lines 503-513", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "lines 533-535", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "lines 487-489", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "line 385", "path": "content.txt", "snapshot_id": "04e4c4a823862a0c2a6d" } ] } }, { "actor_duration_ms": 59573, "duration_ms": 62312, "error": null, "harness_duration_ms": 2739, "human_interventions": 0, "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 55611, "cache_read_input_tokens": 0, "cost_usd": 0.47266900000000006, "input_tokens": 55612, "output_tokens": 6655, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "3cbfff1560fce140026bcd7c12a1be9180eb808291d7a7c243269d622bef66a3", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 305-315", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 334-349", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 149-160", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 460-473", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 46-49", "path": "files/packages/react/src/TextInput/TextInput.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 621-622", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "lines 535-544", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "lines 503-513", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "lines 533-535", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "line 385", "path": "content.txt", "snapshot_id": "04e4c4a823862a0c2a6d" } ] } }, { "actor_duration_ms": 86133, "duration_ms": 118538, "error": null, "harness_duration_ms": 32405, "human_interventions": 0, "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 51415, "cache_read_input_tokens": 4196, "cost_usd": 0.48794180000000004, "input_tokens": 55612, "output_tokens": 9268, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "9b224b2dc328fd61e31c2a67395346fb096827afc754ea5f30a0738763b1cb32", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "failed", "interaction": "failed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [ "accessibility", "interaction" ], "missing_required_props": [], "passed": false, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 305-315", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 334-349", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 149-160", "path": "files/packages/react/src/FormControl/__tests__/FormControl.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 37-40", "path": "files/packages/react/src/TextInput/TextInput.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 46-49", "path": "files/packages/react/src/TextInput/TextInput.test.tsx", "snapshot_id": "25d93d131a1dc240928e" }, { "locator": "lines 503-513", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "lines 533-535", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "line 489", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "lines 621-623", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "lines 535-544", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "line 137", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "line 385", "path": "content.txt", "snapshot_id": "04e4c4a823862a0c2a6d" } ] } } ], "stale_runs": 0, "successful_runs": 2, "task_completion_rate": 1.0, "task_id": "primer-repository-name-field-behavior", "task_success_rate": 0.6667, "type": "component-behavior-implementation" }, { "area": "structure", "attempted_runs": 3, "completed_runs": 3, "exact_answer_reproducibility": 0.3333, "expected_runs": 3, "outcome_agreement": 1.0, "runs": [ { "actor_duration_ms": 164116, "duration_ms": 169502, "error": null, "harness_duration_ms": 5386, "human_interventions": 0, "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 92470, "cache_read_input_tokens": 0, "cost_usd": 0.89735, "input_tokens": 92471, "output_tokens": 18313, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "9f812276895a8dcb55e7ca77eeedfc11adaa41bf67b9baa247c928a32b4ae112", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 385-397", "path": "content.txt", "snapshot_id": "3817f5910fb4bfa7835e" }, { "locator": "lines 1762-1783", "path": "content.txt", "snapshot_id": "3817f5910fb4bfa7835e" }, { "locator": "lines 397-408", "path": "content.txt", "snapshot_id": "04e0d3e95a42896c031c" }, { "locator": "lines 621-622", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "lines 660-662", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "lines 373-380", "path": "content.txt", "snapshot_id": "853c59dd3ecf59e28121" }, { "locator": "lines 471-472", "path": "content.txt", "snapshot_id": "853c59dd3ecf59e28121" }, { "locator": "lines 373-374", "path": "content.txt", "snapshot_id": "230a179f4b704e33f45b" }, { "locator": "lines 386-397", "path": "content.txt", "snapshot_id": "230a179f4b704e33f45b" }, { "locator": "lines 400-402", "path": "content.txt", "snapshot_id": "1ec41a6a82db3c220a80" }, { "locator": "line 360", "path": "content.txt", "snapshot_id": "80ebd2b43e8b45cb676d" }, { "locator": "lines 381-392", "path": "content.txt", "snapshot_id": "80ebd2b43e8b45cb676d" }, { "locator": "lines 503-515", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "lines 383-389", "path": "content.txt", "snapshot_id": "04e4c4a823862a0c2a6d" } ] } }, { "actor_duration_ms": 142135, "duration_ms": 147436, "error": null, "harness_duration_ms": 5301, "human_interventions": 0, "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 88303, "cache_read_input_tokens": 4195, "cost_usd": 0.8408765, "input_tokens": 92499, "output_tokens": 16131, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "7c4ff3615391fbe0dd2d84a8ae6d4edb5f176106fc3538f95274f62ece9f0fe3", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 384-399", "path": "content.txt", "snapshot_id": "3817f5910fb4bfa7835e" }, { "locator": "lines 1762-1770", "path": "content.txt", "snapshot_id": "3817f5910fb4bfa7835e" }, { "locator": "lines 397-408", "path": "content.txt", "snapshot_id": "04e0d3e95a42896c031c" }, { "locator": "lines 414-416", "path": "content.txt", "snapshot_id": "04e0d3e95a42896c031c" }, { "locator": "lines 621-622", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "line 678", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "lines 373-380", "path": "content.txt", "snapshot_id": "853c59dd3ecf59e28121" }, { "locator": "lines 486-487", "path": "content.txt", "snapshot_id": "853c59dd3ecf59e28121" }, { "locator": "lines 386-397", "path": "content.txt", "snapshot_id": "230a179f4b704e33f45b" }, { "locator": "lines 397-404", "path": "content.txt", "snapshot_id": "1ec41a6a82db3c220a80" }, { "locator": "line 360", "path": "content.txt", "snapshot_id": "80ebd2b43e8b45cb676d" }, { "locator": "lines 381-392", "path": "content.txt", "snapshot_id": "80ebd2b43e8b45cb676d" }, { "locator": "lines 499-515", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "line 365", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "lines 453-462", "path": "content.txt", "snapshot_id": "ae0faabbdc1e23d922e0" }, { "locator": "lines 383-389", "path": "content.txt", "snapshot_id": "04e4c4a823862a0c2a6d" } ] } }, { "actor_duration_ms": 95010, "duration_ms": 100397, "error": null, "harness_duration_ms": 5387, "human_interventions": 0, "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 88303, "cache_read_input_tokens": 4195, "cost_usd": 0.7553765, "input_tokens": 92499, "output_tokens": 10431, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "d9cc459426e92f4c2066c9ea0bf2909a99de4e19a1572d6234c57213043b4363", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 384-390", "path": "content.txt", "snapshot_id": "3817f5910fb4bfa7835e" }, { "locator": "lines 1762-1783", "path": "content.txt", "snapshot_id": "3817f5910fb4bfa7835e" }, { "locator": "lines 363-389", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "lines 499-517", "path": "content.txt", "snapshot_id": "53523b7aa25b8b2612cf" }, { "locator": "lines 397-408", "path": "content.txt", "snapshot_id": "04e0d3e95a42896c031c" }, { "locator": "lines 412-437", "path": "content.txt", "snapshot_id": "04e0d3e95a42896c031c" }, { "locator": "lines 476-488", "path": "content.txt", "snapshot_id": "04e0d3e95a42896c031c" }, { "locator": "lines 613-678", "path": "content.txt", "snapshot_id": "add9c1967361d4f74f06" }, { "locator": "lines 373-380", "path": "content.txt", "snapshot_id": "853c59dd3ecf59e28121" }, { "locator": "lines 464-497", "path": "content.txt", "snapshot_id": "853c59dd3ecf59e28121" }, { "locator": "lines 373-374", "path": "content.txt", "snapshot_id": "230a179f4b704e33f45b" }, { "locator": "lines 536-604", "path": "content.txt", "snapshot_id": "230a179f4b704e33f45b" }, { "locator": "lines 375-376", "path": "content.txt", "snapshot_id": "1ec41a6a82db3c220a80" }, { "locator": "lines 393-411", "path": "content.txt", "snapshot_id": "1ec41a6a82db3c220a80" }, { "locator": "lines 359-360", "path": "content.txt", "snapshot_id": "80ebd2b43e8b45cb676d" }, { "locator": "lines 381-392", "path": "content.txt", "snapshot_id": "80ebd2b43e8b45cb676d" }, { "locator": "lines 411-422", "path": "content.txt", "snapshot_id": "80ebd2b43e8b45cb676d" }, { "locator": "lines 429-437", "path": "content.txt", "snapshot_id": "ae0faabbdc1e23d922e0" }, { "locator": "lines 381-391", "path": "content.txt", "snapshot_id": "04e4c4a823862a0c2a6d" } ] } } ], "stale_runs": 0, "successful_runs": 3, "task_completion_rate": 1.0, "task_id": "primer-create-configure-save-flow", "task_success_rate": 1.0, "type": "flow-composition" }, { "area": "governance", "attempted_runs": 3, "completed_runs": 3, "exact_answer_reproducibility": 0.3333, "expected_runs": 3, "outcome_agreement": 1.0, "runs": [ { "actor_duration_ms": 76281, "duration_ms": 76363, "error": null, "harness_duration_ms": 82, "human_interventions": 0, "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 25257, "cache_read_input_tokens": 0, "cost_usd": 0.291351, "input_tokens": 25258, "output_tokens": 8272, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "a15776256e7cd436f5f8dcf6e68d7b6898dc9d00998887bb9f1db049a476ccd0", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "governance-process": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 410-412", "path": "content.txt", "snapshot_id": "51ca77b7640ae2ad9db1" }, { "locator": "lines 40-42", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 371-377", "path": "content.txt", "snapshot_id": "4066f07b7f661cc381e7" }, { "locator": "lines 361-368", "path": "content.txt", "snapshot_id": "b0ecf70d6ccc6d2328dc" }, { "locator": "lines 261-270", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 247-251", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 27-43", "path": "files/.github/workflows/check_for_changeset.yml", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 23-30", "path": "files/.github/pull_request_template.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 257-272", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 367-369", "path": "content.txt", "snapshot_id": "4066f07b7f661cc381e7" } ] } }, { "actor_duration_ms": 61611, "duration_ms": 61682, "error": null, "harness_duration_ms": 71, "human_interventions": 0, "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 20987, "cache_read_input_tokens": 4194, "cost_usd": 0.2446892, "input_tokens": 25182, "output_tokens": 6785, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "4b122bb68a3107fc7acbbdaa84058652fb85a64fe569cd3d7e789329d2a723ba", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "governance-process": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 390-392", "path": "content.txt", "snapshot_id": "51ca77b7640ae2ad9db1" }, { "locator": "lines 40-42", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 371-377", "path": "content.txt", "snapshot_id": "4066f07b7f661cc381e7" }, { "locator": "lines 361-368", "path": "content.txt", "snapshot_id": "b0ecf70d6ccc6d2328dc" }, { "locator": "lines 247-255", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 261-270", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 23-30", "path": "files/.github/pull_request_template.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 257-259", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 1-13", "path": "files/.github/workflows/check_for_changeset.yml", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "line 272", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 367-369", "path": "content.txt", "snapshot_id": "4066f07b7f661cc381e7" } ] } }, { "actor_duration_ms": 39500, "duration_ms": 39558, "error": null, "harness_duration_ms": 58, "human_interventions": 0, "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 20987, "cache_read_input_tokens": 4194, "cost_usd": 0.2044792, "input_tokens": 25182, "output_tokens": 4104, "uncached_input_tokens": 1 }, "validation": { "answer_fingerprint": "9b40e15b58660844c00692689653508c79f3047d41a24858bd37b3c966687314", "citation_locator_validity": 1.0, "citation_normalizations": [], "citation_oracle_accuracy": 1.0, "correct_token_reuse_rate": null, "harness_outcomes": { "governance-process": "passed" }, "invalid_citations": [], "invalid_components": [], "invalid_prop_values": [], "invalid_props": [], "invalid_tokens": [], "missing_required": { "behaviors": [], "components": [], "governance_actions": [], "states": [], "steps": [], "tokens": [] }, "missing_required_citation_sources": [], "missing_required_citations": [], "missing_required_harness_outcomes": [], "missing_required_props": [], "passed": true, "raw_values": [], "required_contract_coverage": 1.0, "required_provenance_coverage": 1.0, "task_completed": true, "valid_citations": [ { "locator": "lines 410-412", "path": "content.txt", "snapshot_id": "51ca77b7640ae2ad9db1" }, { "locator": "lines 367-369", "path": "content.txt", "snapshot_id": "4066f07b7f661cc381e7" }, { "locator": "lines 371-377", "path": "content.txt", "snapshot_id": "4066f07b7f661cc381e7" }, { "locator": "lines 361-368", "path": "content.txt", "snapshot_id": "b0ecf70d6ccc6d2328dc" }, { "locator": "lines 247-255", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 261-270", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 257-272", "path": "files/contributor-docs/CONTRIBUTING.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 23-30", "path": "files/.github/pull_request_template.md", "snapshot_id": "afbb7a337d452b0d0a14" }, { "locator": "lines 1-44", "path": "files/.github/workflows/check_for_changeset.yml", "snapshot_id": "afbb7a337d452b0d0a14" } ] } } ], "stale_runs": 0, "successful_runs": 3, "task_completion_rate": 1.0, "task_id": "primer-governed-breaking-change", "task_success_rate": 1.0, "type": "governed-change-proposal" } ] }