{ "actor": { "model": "claude-sonnet-5", "profile_id": "claude-code/claude-sonnet-5/cloudscape-confirmatory-controlled-v1", "provider": "claude-code", "response_parser": "claude-json" }, "adjudication_sha256": "e0b4e1c1fa255d5955dea2bf3e76d913e641e9324a821c55e85a36e0589e5dff", "attempted_runs": 60, "claim_boundary": "Full benchmark-contract compliance under two operational profiles. This purposive task set does not estimate representative implementation reliability, and the profile difference does not identify documentation causality.", "complete": true, "completed_runs": 60, "condition_id": "controlled", "condition_label": "Frozen evidence only", "evidence_compliance": { "full_pass_rate": 0.6833333333333333, "full_passes": 41, "mean_locator_validity": 0.997295, "mean_required_region_coverage": 0.81667, "mean_required_source_coverage": 1.0, "runs": 60 }, "expected_runs": 60, "measurement_kind_results": { "contract-transcription": { "pass_rate": 0.5555555555555556, "passes": 10, "runs": 18 }, "executable-software": { "pass_rate": 0.3055555555555556, "passes": 11, "runs": 36 }, "governance-procedure": { "pass_rate": 1.0, "passes": 3, "runs": 3 }, "semantic-selection": { "pass_rate": 0.0, "passes": 0, "runs": 3 } }, "metrics": { "cache_creation_input_tokens": 1613155, "cache_read_input_tokens": 498427, "citation_locator_validity": 0.997295, "citation_oracle_accuracy": 0.81667, "cost_usd": 15.537983100000002, "full_evidence_compliance_passes": 41, "full_evidence_compliance_rate": 0.6833333333333333, "human_interventions": 0, "input_tokens": 2111668, "output_tokens": 311251, "required_provenance_coverage": 1.0, "task_success_rate": 0.4, "uncached_input_tokens": 86 }, "publication_issues": [ "The Cloudscape agent-publication hold is active.", "Independent final review has not issued a go decision." ], "publication_ready": false, "report_fingerprint": "b554b37faa9b7cac20105b9dc8e7349ba2355dcec0cfcd5e5ff1cfce8a25689e", "report_format": "ds-eval/agent-performance-report@2", "result_version": "offline-v2", "stale_runs": 0, "suite": "design-system-agent-tasks/1.0.0", "system": "cloudscape", "target_areas": [ "foundations", "bindings", "components", "structure", "governance" ], "task_review_sha256": "c68ff6f3da064d23d176e00339ddcdbf158df723556dbd6fb04bb551ae64cae7", "tasks": [ { "area": "foundations", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "semantic-selection", "measurement_valid": true, "produces_executable_software": false, "public_description": "Identify Cloudscape\u2019s semantic main-content background token in JavaScript and Sass, support both visual modes, avoid hard-coded colors, and cite the governing guidance.", "public_name": "Choose the AppLayout background token for light and dark mode", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "AppLayout.content" ], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.5, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-fail", "measurement_kind": "semantic-selection", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 29522, "cache_read_input_tokens": 0, "cost_usd": 0.218249, "input_tokens": 29523, "output_tokens": 1637, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "AppLayout.content" ], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.5, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-fail", "measurement_kind": "semantic-selection", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 25319, "cache_read_input_tokens": 4203, "cost_usd": 0.19084689999999999, "input_tokens": 29523, "output_tokens": 1407, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.5, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-fail", "measurement_kind": "semantic-selection", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 26816, "cache_read_input_tokens": 33725, "cost_usd": 0.22112149999999997, "input_tokens": 60544, "output_tokens": 2236, "uncached_input_tokens": 3 } } ], "successful_runs": 0, "task_id": "cloudscape-visual-mode-token-mapping-v2", "task_success_rate": 0.0, "type": "semantic-token-selection" }, { "area": "bindings", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Build a typed controlled Table with stable row identity, named selection controls, and a browser-rendered selection update.", "public_name": "Connect single-row Table selection to React state", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "executable-software", "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 35732, "cache_read_input_tokens": 0, "cost_usd": 0.29569900000000005, "input_tokens": 35733, "output_tokens": 3920, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "executable-software", "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 31531, "cache_read_input_tokens": 4201, "cost_usd": 0.27836330000000004, "input_tokens": 35733, "output_tokens": 4361, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "executable-software", "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 31531, "cache_read_input_tokens": 4201, "cost_usd": 0.2715633, "input_tokens": 35733, "output_tokens": 3907, "uncached_input_tokens": 1 } } ], "successful_runs": 3, "task_id": "cloudscape-table-selection-api-binding-v2", "task_success_rate": 1.0, "type": "design-to-code-binding" }, { "area": "components", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Use controlled Toggle state, update it from the Cloudscape change event, announce the preference, and keep it usable at 320 pixels.", "public_name": "Build a keyboard-operable automatic-updates toggle", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "CloudscapeControlledPreference.default_export" ], "missing": { "behaviors": [ "announce-preference-status", "controlled-detail-checked-update", "keyboard-toggle", "usable-at-320px" ], "states": [ "updates-disabled", "updates-enabled" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 21174, "cache_read_input_tokens": 0, "cost_usd": 0.19130999999999998, "input_tokens": 21175, "output_tokens": 3524, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "announce-preference-status", "controlled-detail-checked-update", "keyboard-toggle", "usable-at-320px" ], "states": [ "updates-disabled", "updates-enabled" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.6667, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 19198, "cache_read_input_tokens": 25373, "cost_usd": 0.1976469, "input_tokens": 44574, "output_tokens": 4229, "uncached_input_tokens": 3 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "announce-preference-status", "controlled-detail-checked-update", "keyboard-toggle", "usable-at-320px" ], "states": [ "updates-disabled", "updates-enabled" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "executable-software", "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 22515, "cache_read_input_tokens": 49944, "cost_usd": 0.2723412, "input_tokens": 72464, "output_tokens": 7390, "uncached_input_tokens": 5 } } ], "successful_runs": 1, "task_id": "cloudscape-controlled-toggle-preference", "task_success_rate": 0.3333333333333333, "type": "component-behavior-implementation" }, { "area": "structure", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Compose AppLayout, Table, Input, and Button so an empty collection can create a resource and a filtered no-match state does not show the create action.", "public_name": "Build empty, populated, and filtered inventory states", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "Input.placeholder" ], "missing": { "behaviors": [ "create-from-empty-collection", "filter-resources", "hide-create-in-no-match", "usable-at-320px" ], "states": [ "empty-collection", "filtered-no-match", "populated" ], "steps": [ "create-resource", "filter-to-no-match", "view-empty-inventory", "view-populated-inventory" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.5, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "evaluator-false-positive", "measurement_kind": "executable-software", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 72791, "cache_read_input_tokens": 0, "cost_usd": 0.593034, "input_tokens": 72792, "output_tokens": 7064, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "Input.placeholder", "Input.type" ], "missing": { "behaviors": [ "create-from-empty-collection", "filter-resources", "hide-create-in-no-match", "usable-at-320px" ], "states": [ "empty-collection", "filtered-no-match", "populated" ], "steps": [ "create-resource", "filter-to-no-match", "view-empty-inventory", "view-populated-inventory" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 0.9286, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.5, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 68588, "cache_read_input_tokens": 4203, "cost_usd": 0.5640719, "input_tokens": 72792, "output_tokens": 6730, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "Input.placeholder", "Table.filter" ], "missing": { "behaviors": [ "create-from-empty-collection", "filter-resources", "hide-create-in-no-match", "usable-at-320px" ], "states": [ "empty-collection", "filtered-no-match", "populated" ], "steps": [ "create-resource", "filter-to-no-match", "view-empty-inventory", "view-populated-inventory" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.5, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "evaluator-false-positive", "measurement_kind": "executable-software", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 68588, "cache_read_input_tokens": 4203, "cost_usd": 0.6161319, "input_tokens": 72792, "output_tokens": 10202, "uncached_input_tokens": 1 } } ], "successful_runs": 0, "task_id": "cloudscape-resource-inventory-empty-state-v2", "task_success_rate": 0.0, "type": "flow-composition" }, { "area": "structure", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Validate and focus a required name, configure an option, save the reviewed payload, show progress, and recover from a rejected save.", "public_name": "Build create, configure, save, success, and retry states", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "AppLayout.contentType", "AppLayout.notifications", "Button.loading", "Button.loadingText", "Spinner.size" ], "missing": { "behaviors": [ "focus-first-invalid-on-continue", "reject-to-recovery", "resolve-to-success", "retry-save", "save-reviewed-payload", "usable-at-320px" ], "states": [ "create", "failure", "saving" ], "steps": [ "configure", "create", "save", "success-or-recovery", "validate" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 50565, "cache_read_input_tokens": 0, "cost_usd": 0.575042, "input_tokens": 50566, "output_tokens": 15815, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "AppLayout.contentType", "AppLayout.notifications", "Button.note", "Input.ariaRequired" ], "missing": { "behaviors": [ "focus-first-invalid-on-continue", "reject-to-recovery", "resolve-to-success", "retry-save", "save-reviewed-payload", "usable-at-320px" ], "states": [ "configure", "create", "failure", "saving", "success" ], "steps": [ "configure", "create", "save", "success-or-recovery", "validate" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "failed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 46367, "cache_read_input_tokens": 4198, "cost_usd": 0.5233634, "input_tokens": 50566, "output_tokens": 13965, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "AppLayout.contentType", "AppLayout.notifications", "Button.context" ], "missing": { "behaviors": [ "focus-first-invalid-on-continue", "reject-to-recovery", "resolve-to-success", "retry-save", "save-reviewed-payload", "usable-at-320px" ], "states": [ "create", "failure", "saving" ], "steps": [ "configure", "create", "save", "success-or-recovery", "validate" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 46367, "cache_read_input_tokens": 4198, "cost_usd": 0.5184633999999999, "input_tokens": 50566, "output_tokens": 13639, "uncached_input_tokens": 1 } } ], "successful_runs": 0, "task_id": "cloudscape-create-configure-save-flow", "task_success_rate": 0.0, "type": "flow-composition" }, { "area": "governance", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "contract-transcription", "measurement_valid": true, "produces_executable_software": false, "public_description": "Order the documented test, visual, bundle-size, ownership, and release-note evidence, and leave undocumented deployment approval explicitly unknown.", "public_name": "Plan the public evidence needed to release a change", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.6667, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-fail", "measurement_kind": "contract-transcription", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 38146, "cache_read_input_tokens": 0, "cost_usd": 0.323736, "input_tokens": 38147, "output_tokens": 4703, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.6667, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-fail", "measurement_kind": "contract-transcription", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 33946, "cache_read_input_tokens": 4200, "cost_usd": 0.30213599999999996, "input_tokens": 38147, "output_tokens": 4858, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.6667, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-fail", "measurement_kind": "contract-transcription", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 33946, "cache_read_input_tokens": 4200, "cost_usd": 0.30572599999999994, "input_tokens": 38147, "output_tokens": 5099, "uncached_input_tokens": 1 } } ], "successful_runs": 0, "task_id": "cloudscape-release-evidence-governance-v2", "task_success_rate": 0.0, "type": "governed-change-proposal" }, { "area": "foundations", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "contract-transcription", "measurement_valid": true, "produces_executable_software": false, "public_description": "Select Cloudscape\u2019s semantic focus-border token in JavaScript and Sass, avoid raw color values, and cite the frozen token source.", "public_name": "Choose the token for a visible keyboard focus ring", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "contract-transcription", "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 29504, "cache_read_input_tokens": 0, "cost_usd": 0.223004, "input_tokens": 29505, "output_tokens": 1962, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "contract-transcription", "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 26201, "cache_read_input_tokens": 33708, "cost_usd": 0.2040744, "input_tokens": 59912, "output_tokens": 1347, "uncached_input_tokens": 3 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "contract-transcription", "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 27788, "cache_read_input_tokens": 95243, "cost_usd": 0.2609489, "input_tokens": 123038, "output_tokens": 3272, "uncached_input_tokens": 7 } } ], "successful_runs": 3, "task_id": "cloudscape-focus-visible-semantic-pair-v1", "task_success_rate": 1.0, "type": "semantic-token-selection" }, { "area": "foundations", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "contract-transcription", "measurement_valid": true, "produces_executable_software": false, "public_description": "Choose Cloudscape\u2019s element-query strategy and scaled large-spacing token without inventing pixel values or media queries.", "public_name": "Use Cloudscape\u2019s responsive spacing system", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "contract-transcription", "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 16577, "cache_read_input_tokens": 0, "cost_usd": 0.11832400000000001, "input_tokens": 16578, "output_tokens": 741, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "contract-transcription", "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 12376, "cache_read_input_tokens": 4201, "cost_usd": 0.1057033, "input_tokens": 16578, "output_tokens": 1496, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "contract-transcription", "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 16577, "cache_read_input_tokens": 0, "cost_usd": 0.12334900000000001, "input_tokens": 16578, "output_tokens": 1076, "uncached_input_tokens": 1 } } ], "successful_runs": 3, "task_id": "cloudscape-responsive-content-spacing-v1", "task_success_rate": 1.0, "type": "semantic-token-selection" }, { "area": "foundations", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "contract-transcription", "measurement_valid": true, "produces_executable_software": false, "public_description": "Pair semantic body-text and main-background tokens for light and dark mode, and avoid direct CSS overrides of Cloudscape components.", "public_name": "Style a custom surface without hard-coded colors", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": false, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "contract-transcription", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 15920, "cache_read_input_tokens": 0, "cost_usd": 0.151314, "input_tokens": 15921, "output_tokens": 3227, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": false, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "contract-transcription", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 11717, "cache_read_input_tokens": 4203, "cost_usd": 0.13467690000000002, "input_tokens": 15921, "output_tokens": 3715, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "strict" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "semantic-selection": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": false, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "contract-transcription", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 11717, "cache_read_input_tokens": 4203, "cost_usd": 0.11572690000000001, "input_tokens": 15921, "output_tokens": 2451, "uncached_input_tokens": 1 } } ], "successful_runs": 0, "task_id": "cloudscape-dark-surface-token-boundary-v1", "task_success_rate": 0.0, "type": "semantic-token-selection" }, { "area": "bindings", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Build a typed Modal with an explicit header, footer actions, controlled visibility, and working open and Cancel behavior.", "public_name": "Connect a controlled Modal to its open and close actions", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "executable-software", "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 31062, "cache_read_input_tokens": 0, "cost_usd": 0.268977, "input_tokens": 31063, "output_tokens": 4214, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "executable-software", "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 26863, "cache_read_input_tokens": 4199, "cost_usd": 0.2754777, "input_tokens": 31063, "output_tokens": 6244, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "Button.role", "Button.role", "Button.role", "SpaceBetween.direction", "SpaceBetween.size" ], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 31062, "cache_read_input_tokens": 0, "cost_usd": 0.263497, "input_tokens": 31063, "output_tokens": 3850, "uncached_input_tokens": 1 } } ], "successful_runs": 2, "task_id": "cloudscape-modal-api-binding-v2", "task_success_rate": 0.6666666666666666, "type": "design-to-code-binding" }, { "area": "components", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Use Flashbar and Button to show a load error, expose a retry state, support dismissal, pass accessibility checks, and fit a narrow viewport.", "public_name": "Build a retryable and dismissible error message", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "dismiss-removes-message", "retry-updates-content", "usable-at-320px" ], "states": [ "dismissed", "load-failure", "retrying" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.6667, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 29976, "cache_read_input_tokens": 0, "cost_usd": 0.310253, "input_tokens": 29977, "output_tokens": 7461, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "dismiss-removes-message", "retry-updates-content", "usable-at-320px" ], "states": [ "dismissed", "load-failure", "retrying" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 25774, "cache_read_input_tokens": 4202, "cost_usd": 0.2839366, "input_tokens": 29977, "output_tokens": 7303, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "dismiss-removes-message", "retry-updates-content", "usable-at-320px" ], "states": [ "dismissed", "load-failure", "retrying" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 0.6667, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 25774, "cache_read_input_tokens": 4202, "cost_usd": 0.26129660000000005, "input_tokens": 29977, "output_tokens": 5794, "uncached_input_tokens": 1 } } ], "successful_runs": 0, "task_id": "cloudscape-flashbar-recovery-message-v2", "task_success_rate": 0.0, "type": "component-behavior-implementation" }, { "area": "governance", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "contract-transcription", "measurement_valid": true, "produces_executable_software": false, "public_description": "Order the public issue, focused accessibility and visual tests, reviewed change, continuous-integration checks, and release evidence.", "public_name": "Plan an accessibility regression fix from report to release", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-pass", "measurement_kind": "contract-transcription", "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 22193, "cache_read_input_tokens": 0, "cost_usd": 0.200293, "input_tokens": 22194, "output_tokens": 3685, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 0.9091, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-fail", "measurement_kind": "contract-transcription", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 22638, "cache_read_input_tokens": 51307, "cost_usd": 0.2567371, "input_tokens": 73950, "output_tokens": 6243, "uncached_input_tokens": 5 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-pass", "measurement_kind": "contract-transcription", "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 22193, "cache_read_input_tokens": 0, "cost_usd": 0.219843, "input_tokens": 22194, "output_tokens": 4988, "uncached_input_tokens": 1 } } ], "successful_runs": 2, "task_id": "cloudscape-accessibility-regression-governance-v2", "task_success_rate": 0.6666666666666666, "type": "governed-change-proposal" }, { "area": "bindings", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Keep activeHref in React state, prevent full-page navigation, and update the active item from the Cloudscape follow event.", "public_name": "Update SideNavigation after an internal link is followed", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [ "SideNavigation.activeHref" ], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 25318, "cache_read_input_tokens": 0, "cost_usd": 0.214353, "input_tokens": 25319, "output_tokens": 3164, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [ "SideNavigation.activeHref" ], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 25984, "cache_read_input_tokens": 29519, "cost_usd": 0.2780707, "input_tokens": 55506, "output_tokens": 6557, "uncached_input_tokens": 3 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [ "SideNavigation.activeHref" ], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 21117, "cache_read_input_tokens": 4201, "cost_usd": 0.19563730000000001, "input_tokens": 25319, "output_tokens": 3515, "uncached_input_tokens": 1 } } ], "successful_runs": 0, "task_id": "cloudscape-navigation-follow-binding-v3", "task_success_rate": 0.0, "type": "design-to-code-binding" }, { "area": "bindings", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Build a typed FormField and Input pair whose error clears after the user enters a resource name.", "public_name": "Connect a required name field to validation state", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "evaluator-false-negative", "measurement_kind": "executable-software", "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 19345, "cache_read_input_tokens": 0, "cost_usd": 0.160839, "input_tokens": 19346, "output_tokens": 2295, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 15145, "cache_read_input_tokens": 4200, "cost_usd": 0.138069, "input_tokens": 19346, "output_tokens": 2373, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "render": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "evaluator-false-negative", "measurement_kind": "executable-software", "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 15145, "cache_read_input_tokens": 4200, "cost_usd": 0.130599, "input_tokens": 19346, "output_tokens": 1875, "uncached_input_tokens": 1 } } ], "successful_runs": 2, "task_id": "cloudscape-required-name-input-binding-v2", "task_success_rate": 0.6666666666666666, "type": "design-to-code-binding" }, { "area": "components", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Open a controlled Modal, close it through the Cancel action and onDismiss path, pass accessibility checks, and fit a narrow viewport.", "public_name": "Make both Cancel and Escape close a Modal", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "Modal.closeLabel" ], "missing": { "behaviors": [ "cancel-dismisses", "on-dismiss-closes", "open-modal", "usable-at-320px" ], "states": [ "closed", "closed-again", "open" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 24969, "cache_read_input_tokens": 0, "cost_usd": 0.22036, "input_tokens": 24970, "output_tokens": 3747, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "cancel-dismisses", "on-dismiss-closes", "open-modal", "usable-at-320px" ], "states": [ "closed", "closed-again", "open" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "executable-software", "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 20767, "cache_read_input_tokens": 4202, "cost_usd": 0.1982836, "input_tokens": 24970, "output_tokens": 3872, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "Modal.closeLabel" ], "missing": { "behaviors": [ "cancel-dismisses", "on-dismiss-closes", "open-modal", "usable-at-320px" ], "states": [ "closed", "closed-again", "open" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 26412, "cache_read_input_tokens": 57768, "cost_usd": 0.29962540000000004, "input_tokens": 84185, "output_tokens": 7298, "uncached_input_tokens": 5 } } ], "successful_runs": 1, "task_id": "cloudscape-modal-open-close-behavior-v3", "task_success_rate": 0.3333333333333333, "type": "component-behavior-implementation" }, { "area": "structure", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Filter Alpha and Beta rows, show a distinct no-match state, and restore the full inventory when the filter is cleared.", "public_name": "Filter a resource table and recover from no matches", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "FormField.label", "Table.filter", "Table.resizableColumns" ], "missing": { "behaviors": [ "clear-restores-rows", "filter-matching-rows", "show-no-match-state", "usable-at-320px" ], "states": [ "cleared", "filtered", "no-match", "unfiltered" ], "steps": [ "clear-filter", "filter-match", "filter-no-match", "show-inventory" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 39447, "cache_read_input_tokens": 0, "cost_usd": 0.427719, "input_tokens": 39448, "output_tokens": 11013, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "Box.textAlign", "Box.variant" ], "missing": { "behaviors": [ "clear-restores-rows", "filter-matching-rows", "show-no-match-state", "usable-at-320px" ], "states": [ "cleared", "filtered", "no-match", "unfiltered" ], "steps": [ "clear-filter", "filter-match", "filter-no-match", "show-inventory" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 35243, "cache_read_input_tokens": 4204, "cost_usd": 0.36293620000000004, "input_tokens": 39448, "output_tokens": 8292, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "Box.id", "Input.ariaLabelledby" ], "missing": { "behaviors": [ "clear-restores-rows", "filter-matching-rows", "show-no-match-state", "usable-at-320px" ], "states": [ "cleared", "filtered", "no-match", "unfiltered" ], "steps": [ "clear-filter", "filter-match", "filter-no-match", "show-inventory" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 35243, "cache_read_input_tokens": 4204, "cost_usd": 0.38851119999999995, "input_tokens": 39448, "output_tokens": 9996, "uncached_input_tokens": 1 } } ], "successful_runs": 0, "task_id": "cloudscape-filtered-inventory-empty-state-v1", "task_success_rate": 0.0, "type": "flow-composition" }, { "area": "governance", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "contract-transcription", "measurement_valid": true, "produces_executable_software": false, "public_description": "Document the defect, add a focused accessibility test, pass review and continuous integration, and retain release evidence.", "public_name": "Plan the review path for an accessibility fix", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-fail", "measurement_kind": "contract-transcription", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 22157, "cache_read_input_tokens": 0, "cost_usd": 0.191833, "input_tokens": 22158, "output_tokens": 3137, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-pass", "measurement_kind": "contract-transcription", "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 17953, "cache_read_input_tokens": 4204, "cost_usd": 0.1751702, "input_tokens": 22158, "output_tokens": 3624, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-pass", "measurement_kind": "contract-transcription", "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 17953, "cache_read_input_tokens": 4204, "cost_usd": 0.1595852, "input_tokens": 22158, "output_tokens": 2585, "uncached_input_tokens": 1 } } ], "successful_runs": 2, "task_id": "cloudscape-accessibility-fix-review-governance-v1", "task_success_rate": 0.6666666666666666, "type": "governed-change-proposal" }, { "area": "components", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Use a Cloudscape Button to expose idle, pending, and completed states while preventing duplicate submission.", "public_name": "Show loading, disable Save, then show Saved", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "disable-and-load-while-pending", "show-saved-label", "start-pending", "usable-at-320px" ], "states": [ "idle", "pending", "saved" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "executable-software", "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 16110, "cache_read_input_tokens": 0, "cost_usd": 0.16360000000000002, "input_tokens": 16111, "output_tokens": 3942, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "disable-and-load-while-pending", "show-saved-label", "start-pending", "usable-at-320px" ], "states": [ "idle", "pending", "saved" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "evaluator-false-positive", "measurement_kind": "executable-software", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 11908, "cache_read_input_tokens": 4202, "cost_usd": 0.1406386, "input_tokens": 16111, "output_tokens": 4008, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "disable-and-load-while-pending", "show-saved-label", "start-pending", "usable-at-320px" ], "states": [ "idle", "pending", "saved" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "confirmed-pass", "measurement_kind": "executable-software", "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 11908, "cache_read_input_tokens": 4202, "cost_usd": 0.1528486, "input_tokens": 16111, "output_tokens": 4822, "uncached_input_tokens": 1 } } ], "successful_runs": 2, "task_id": "cloudscape-button-pending-label-behavior-v2", "task_success_rate": 0.6666666666666666, "type": "component-behavior-implementation" }, { "area": "structure", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "executable-software", "measurement_valid": true, "produces_executable_software": true, "public_description": "Keep Alpha visible while loading, render a visible Cloudscape Spinner beside the status, then add Beta without prescribing a timer implementation.", "public_name": "Preserve rows, show a Spinner, and append loaded results", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [ "LiveRegion.hidden" ], "missing": { "behaviors": [ "add-beta-on-success", "preserve-alpha-while-loading", "show-spinner", "usable-at-320px" ], "states": [ "initial", "loaded", "loading" ], "steps": [ "complete-with-beta", "preserve-alpha", "show-alpha", "start-load" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 0.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 22558, "cache_read_input_tokens": 0, "cost_usd": 0.289001, "input_tokens": 22559, "output_tokens": 9404, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "add-beta-on-success", "preserve-alpha-while-loading", "show-spinner", "usable-at-320px" ], "states": [ "initial", "loaded", "loading" ], "steps": [ "complete-with-beta", "preserve-alpha", "show-alpha", "start-load" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "passed", "interaction": "passed", "responsive": "passed", "typecheck": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 0.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "passed" }, "final_disposition": "evaluator-false-positive", "measurement_kind": "executable-software", "passed": false, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 18358, "cache_read_input_tokens": 4200, "cost_usd": 0.23719099999999999, "input_tokens": 22559, "output_tokens": 7546, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": { "behaviors": [ "add-beta-on-success", "preserve-alpha-while-loading", "show-spinner", "usable-at-320px" ], "states": [ "initial", "loaded", "loading" ], "steps": [ "complete-with-beta", "preserve-alpha", "show-alpha", "start-load" ] }, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": false, "blocking_contract_complete": false, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": false, "governance_accuracy": "not-applicable", "harness_outcomes": { "accessibility": "not-run", "interaction": "not-run", "responsive": "not-run", "typecheck": "failed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": false, "required_region_coverage": 0.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "failed" }, "final_disposition": "confirmed-fail", "measurement_kind": "executable-software", "passed": false, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 18358, "cache_read_input_tokens": 4200, "cost_usd": 0.286271, "input_tokens": 22559, "output_tokens": 10818, "uncached_input_tokens": 1 } } ], "successful_runs": 0, "task_id": "cloudscape-load-more-success-flow-v4", "task_success_rate": 0.0, "type": "flow-composition" }, { "area": "governance", "attempted_runs": 3, "completed_runs": 3, "expected_runs": 3, "measurement_kind": "governance-procedure", "measurement_valid": true, "produces_executable_software": false, "public_description": "Construct a review sequence from public build, test, ownership, and contribution evidence without inventing an internal approval step.", "public_name": "Use public repository evidence to plan a reviewed component change", "runs": [ { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-pass", "measurement_kind": "governance-procedure", "passed": true, "run": 1, "status": "complete", "usage": { "cache_creation_input_tokens": 17201, "cache_read_input_tokens": 0, "cost_usd": 0.158471, "input_tokens": 17202, "output_tokens": 3137, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-pass", "measurement_kind": "governance-procedure", "passed": true, "run": 2, "status": "complete", "usage": { "cache_creation_input_tokens": 13001, "cache_read_input_tokens": 4200, "cost_usd": 0.160256, "input_tokens": 17202, "output_tokens": 4852, "uncached_input_tokens": 1 } }, { "dimensions": { "advisory_response_metadata": { "coverage": 1.0, "invalid_prop_values": [], "invalid_props": [], "missing": {}, "missing_required_props": [], "policy": "advisory" }, "blocking_api_and_token_compliance": true, "blocking_contract_complete": true, "citation_locator_validity": 1.0, "evaluator_agreement": "agreed", "frozen_evidence_compliance": true, "governance_accuracy": "passed", "harness_outcomes": { "governance-process": "passed" }, "harness_validity": "valid", "oracle_validity": "valid", "required_harness_compliance": true, "required_region_coverage": 1.0, "required_source_coverage": 1.0, "response_complete": true, "semantic_or_executable_outcome": "not-applicable" }, "final_disposition": "confirmed-pass", "measurement_kind": "governance-procedure", "passed": true, "run": 3, "status": "complete", "usage": { "cache_creation_input_tokens": 13001, "cache_read_input_tokens": 4200, "cost_usd": 0.171836, "input_tokens": 17202, "output_tokens": 5624, "uncached_input_tokens": 1 } } ], "successful_runs": 3, "task_id": "cloudscape-public-change-review-governance-v3", "task_success_rate": 1.0, "type": "governed-change-proposal" } ], "tool_exposure": { "runs_with_requests": 0, "total_requests": 0 }, "v2_replay_manifest_sha256": "1ca3f605b9cc9a7fde3d4db404780cdea5b8732024dbe7130154c1713ea269f6" }