{ "schema": "simreal.qa.report.v1", "manifest_id": "0afd459fa8e028a9c233493f16f77200fd7ade48a8703b6841e56f53e023523f", "environment_version": "0.3.0", "scope": "synthetic_environment_regression_not_model_benchmark", "generated_at": 1789276293.987136, "total": 288, "passed": 288, "false_positives": 0, "false_negatives": 0, "execution_errors": 0, "all_replays_consistent": true, "source_hashes": { "simreal_envs/__init__.py": "745435a21f3c4bf604ad9db209fa2a15362e128d4ea1649586f62d5c106076bb", "simreal_envs/engine.py": "f5e62fa126ab6d027ce7293279f867f4a9159eae862c342e0922e2f0d2804ed8", "simreal_envs/evaluate.py": "931b5300d78bdbbbb9037be111623fd0d4ec493cecb775c6d45165cd3cf222a1", "simreal_envs/gym_adapter.py": "4885d1ae8f5f03b201aa81c5f8e694eda7907ef653a6e077032d15597aae63df", "simreal_envs/gym_reference.py": "b5b3fdc3cddd8fa4cb056b7847ed0dd39112aeedaf3b20fd3a5d803aecb3b209", "simreal_envs/oracles.py": "1980fd656d397baa772a69e4c15654df477b01345f95e5299a524be2218972cf", "simreal_envs/replay.py": "150614a351163e4d4d44bb806aec03c2c25269938a4f5884c8241d176ebcd894", "simreal_envs/runner.py": "c27b1b626808c3c50df978dccd5c2fe2cd18d89dd46f77179170019fc6a8a2ab", "simreal_envs/scenarios.py": "fe3d230cbc898bf13fcfca1866624a8ba3b4950f27ce84555313d7436501008f", "scripts/js_sandbox.mjs": "482fe98676e8b9dc0b0cec06e441515f2ecaeac1a79d8a2c5b51d7690af8afc6", "package-lock.json": "738ce9b895a2e4da44a2ccd90291a82e98daedd309228cc4c22986704aee852d", "requirements.lock.txt": "0edfaf823fab428fd8cc512e013abdb9f4f9702f14c9ded9838588e53473540c" }, "rows": [ { "domain": "education", "scenario": "同分母加法误区", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.632, "initial_state_hash": "64965a145071907ccc8d1e349a0c2c8aa5de80cca6f5f0665afa28e767cd9df8", "final_state_hash": "acfa6a7a70e29756e9b100aa7561c777026ce7183a15b98a92e6137c1e460fea", "case_id": "education-train-00000" }, { "domain": "education", "scenario": "同分母加法误区", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.232, "initial_state_hash": "64965a145071907ccc8d1e349a0c2c8aa5de80cca6f5f0665afa28e767cd9df8", "final_state_hash": "64965a145071907ccc8d1e349a0c2c8aa5de80cca6f5f0665afa28e767cd9df8", "case_id": "education-train-00001" }, { "domain": "education", "scenario": "同分母加法误区", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.03, "initial_state_hash": "64965a145071907ccc8d1e349a0c2c8aa5de80cca6f5f0665afa28e767cd9df8", "final_state_hash": "64965a145071907ccc8d1e349a0c2c8aa5de80cca6f5f0665afa28e767cd9df8", "case_id": "education-train-00002" }, { "domain": "education", "scenario": "同分母加法误区", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.617, "initial_state_hash": "64965a145071907ccc8d1e349a0c2c8aa5de80cca6f5f0665afa28e767cd9df8", "final_state_hash": "1bc7aac4084117ee4c5ec7f40b4282f2a11ff355f801b3fdbff3c0873c46bdae", "case_id": "education-train-00003" }, { "domain": "education", "scenario": "加法误用乘法", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.469, "initial_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "final_state_hash": "93da6824f5e9aab0d23d33caff744d7d6a90c5f85ab3582d28ade0d2eb951c14", "case_id": "education-train-00004" }, { "domain": "education", "scenario": "加法误用乘法", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.22, "initial_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "final_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "case_id": "education-train-00005" }, { "domain": "education", "scenario": "加法误用乘法", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.942, "initial_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "final_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "case_id": "education-train-00006" }, { "domain": "education", "scenario": "加法误用乘法", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.589, "initial_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "final_state_hash": "cb11572ea65905a7f76aacfca293b2e8b1f01400e050166bb57cb162dd32d3a1", "case_id": "education-train-00007" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.451, "initial_state_hash": "04162ca3ee4165c90d934bbc86caebabe19016718bf2755de95bedebe36a69b7", "final_state_hash": "a2e3a8b68ac5a1bf0acfa2356962e7e1c4f728ab53a4737c4c53526d8fc97efe", "case_id": "education-train-00008" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.214, "initial_state_hash": "04162ca3ee4165c90d934bbc86caebabe19016718bf2755de95bedebe36a69b7", "final_state_hash": "04162ca3ee4165c90d934bbc86caebabe19016718bf2755de95bedebe36a69b7", "case_id": "education-train-00009" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.914, "initial_state_hash": "04162ca3ee4165c90d934bbc86caebabe19016718bf2755de95bedebe36a69b7", "final_state_hash": "04162ca3ee4165c90d934bbc86caebabe19016718bf2755de95bedebe36a69b7", "case_id": "education-train-00010" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.596, "initial_state_hash": "04162ca3ee4165c90d934bbc86caebabe19016718bf2755de95bedebe36a69b7", "final_state_hash": "1ff0c56e6c0c273790ed24634f53563d8aa02099b050c9aa5cf363b36ae3eef0", "case_id": "education-train-00011" }, { "domain": "education", "scenario": "同分母加法误区", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.452, "initial_state_hash": "3c4ba52fa0de4a33aa650dbe8c483662b29fa918f86959e225f7229ec0c253fc", "final_state_hash": "fca96c3abecbdde446adf32fa3cf53ec961505c3491938436be621a8a396935f", "case_id": "education-train-00012" }, { "domain": "education", "scenario": "同分母加法误区", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.213, "initial_state_hash": "3c4ba52fa0de4a33aa650dbe8c483662b29fa918f86959e225f7229ec0c253fc", "final_state_hash": "3c4ba52fa0de4a33aa650dbe8c483662b29fa918f86959e225f7229ec0c253fc", "case_id": "education-train-00013" }, { "domain": "education", "scenario": "同分母加法误区", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.908, "initial_state_hash": "3c4ba52fa0de4a33aa650dbe8c483662b29fa918f86959e225f7229ec0c253fc", "final_state_hash": "3c4ba52fa0de4a33aa650dbe8c483662b29fa918f86959e225f7229ec0c253fc", "case_id": "education-train-00014" }, { "domain": "education", "scenario": "同分母加法误区", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.572, "initial_state_hash": "3c4ba52fa0de4a33aa650dbe8c483662b29fa918f86959e225f7229ec0c253fc", "final_state_hash": "35c3485dd6b13e83c84a52d0b479fd9c4b10d2414e7cf186ebfba6c59b9539cf", "case_id": "education-train-00015" }, { "domain": "education", "scenario": "加法误用乘法", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.449, "initial_state_hash": "7cd2cf04f1be8d4cc66e0483244da51dbe3218f66e47a46cd939d04cbec35f9e", "final_state_hash": "a8ec0c787b7339a0660d48fb275f9ebe46d551313ec2659b21d5037d176f215f", "case_id": "education-train-00016" }, { "domain": "education", "scenario": "加法误用乘法", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.213, "initial_state_hash": "7cd2cf04f1be8d4cc66e0483244da51dbe3218f66e47a46cd939d04cbec35f9e", "final_state_hash": "7cd2cf04f1be8d4cc66e0483244da51dbe3218f66e47a46cd939d04cbec35f9e", "case_id": "education-train-00017" }, { "domain": "education", "scenario": "加法误用乘法", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.932, "initial_state_hash": "7cd2cf04f1be8d4cc66e0483244da51dbe3218f66e47a46cd939d04cbec35f9e", "final_state_hash": "7cd2cf04f1be8d4cc66e0483244da51dbe3218f66e47a46cd939d04cbec35f9e", "case_id": "education-train-00018" }, { "domain": "education", "scenario": "加法误用乘法", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.572, "initial_state_hash": "7cd2cf04f1be8d4cc66e0483244da51dbe3218f66e47a46cd939d04cbec35f9e", "final_state_hash": "152d52095c3bd1f5208cf4b0b7b025a58b155fe3024889599613db5268636367", "case_id": "education-train-00019" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.431, "initial_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "final_state_hash": "cde59a5427122a8e9f8202785429df96fb6690b5ae767b2a7bc47aaf68f46dc2", "case_id": "education-train-00020" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.211, "initial_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "final_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "case_id": "education-train-00021" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.928, "initial_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "final_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "case_id": "education-train-00022" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.576, "initial_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "final_state_hash": "3cdae157e4b186aa9e39669e7ea94e7f4b66fab3db502d6c8ff99fc8bd666348", "case_id": "education-train-00023" }, { "domain": "education", "scenario": "同分母加法误区", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.425, "initial_state_hash": "92f40eeff1a6076acd8dd8105e1cb6889721054af5a9cf4f2e587b12ff37d567", "final_state_hash": "656aa0ac6b4930f470ef7734c899a0220dcc2acaad0ad151a4460a1e76c29e5c", "case_id": "education-eval-00024" }, { "domain": "education", "scenario": "同分母加法误区", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.214, "initial_state_hash": "92f40eeff1a6076acd8dd8105e1cb6889721054af5a9cf4f2e587b12ff37d567", "final_state_hash": "92f40eeff1a6076acd8dd8105e1cb6889721054af5a9cf4f2e587b12ff37d567", "case_id": "education-eval-00025" }, { "domain": "education", "scenario": "同分母加法误区", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.941, "initial_state_hash": "92f40eeff1a6076acd8dd8105e1cb6889721054af5a9cf4f2e587b12ff37d567", "final_state_hash": "92f40eeff1a6076acd8dd8105e1cb6889721054af5a9cf4f2e587b12ff37d567", "case_id": "education-eval-00026" }, { "domain": "education", "scenario": "同分母加法误区", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.57, "initial_state_hash": "92f40eeff1a6076acd8dd8105e1cb6889721054af5a9cf4f2e587b12ff37d567", "final_state_hash": "82d81bc2ec5d5f3306623ec45e1c6589fd1ecc599cf30f4b8fe667d74d0edc95", "case_id": "education-eval-00027" }, { "domain": "education", "scenario": "加法误用乘法", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.421, "initial_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "final_state_hash": "93da6824f5e9aab0d23d33caff744d7d6a90c5f85ab3582d28ade0d2eb951c14", "case_id": "education-eval-00028" }, { "domain": "education", "scenario": "加法误用乘法", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.214, "initial_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "final_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "case_id": "education-eval-00029" }, { "domain": "education", "scenario": "加法误用乘法", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.936, "initial_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "final_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "case_id": "education-eval-00030" }, { "domain": "education", "scenario": "加法误用乘法", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.571, "initial_state_hash": "1fd812c44219c7a7748a16683e813fc0e94e7c0aa5006d9289e52d1009400801", "final_state_hash": "cb11572ea65905a7f76aacfca293b2e8b1f01400e050166bb57cb162dd32d3a1", "case_id": "education-eval-00031" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.432, "initial_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "final_state_hash": "cde59a5427122a8e9f8202785429df96fb6690b5ae767b2a7bc47aaf68f46dc2", "case_id": "education-eval-00032" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.211, "initial_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "final_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "case_id": "education-eval-00033" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.918, "initial_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "final_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "case_id": "education-eval-00034" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.571, "initial_state_hash": "3cf640e01b8588d2cb1595059a1e7989c2f18d464f6ff21f3a2ca2c4c0c7f003", "final_state_hash": "3cdae157e4b186aa9e39669e7ea94e7f4b66fab3db502d6c8ff99fc8bd666348", "case_id": "education-eval-00035" }, { "domain": "education", "scenario": "同分母加法误区", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.431, "initial_state_hash": "aa1d2bf4fd6a2d537f0528758b862dcb3d7e53c5fa8ad89cc1042d8bc71c3dda", "final_state_hash": "ae255c49a7ae7a082c4cf389c5040c4eec3d3c7bcd6e2944ca268212f3ea0246", "case_id": "education-eval-00036" }, { "domain": "education", "scenario": "同分母加法误区", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.212, "initial_state_hash": "aa1d2bf4fd6a2d537f0528758b862dcb3d7e53c5fa8ad89cc1042d8bc71c3dda", "final_state_hash": "aa1d2bf4fd6a2d537f0528758b862dcb3d7e53c5fa8ad89cc1042d8bc71c3dda", "case_id": "education-eval-00037" }, { "domain": "education", "scenario": "同分母加法误区", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.917, "initial_state_hash": "aa1d2bf4fd6a2d537f0528758b862dcb3d7e53c5fa8ad89cc1042d8bc71c3dda", "final_state_hash": "aa1d2bf4fd6a2d537f0528758b862dcb3d7e53c5fa8ad89cc1042d8bc71c3dda", "case_id": "education-eval-00038" }, { "domain": "education", "scenario": "同分母加法误区", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.576, "initial_state_hash": "aa1d2bf4fd6a2d537f0528758b862dcb3d7e53c5fa8ad89cc1042d8bc71c3dda", "final_state_hash": "6883db21421e17d22055496ba8a12bd8ba7c269771fa92079744c42119f3d451", "case_id": "education-eval-00039" }, { "domain": "education", "scenario": "加法误用乘法", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.439, "initial_state_hash": "8a1b390d15abbd744d2db6019a79a058baa8db88ac0daf1c682ff5fe088b249e", "final_state_hash": "378f3e204689bc4fb8a94969fae46b3bcd09cbda9dcd26d86a9a42fa94a9f6d5", "case_id": "education-eval-00040" }, { "domain": "education", "scenario": "加法误用乘法", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.211, "initial_state_hash": "8a1b390d15abbd744d2db6019a79a058baa8db88ac0daf1c682ff5fe088b249e", "final_state_hash": "8a1b390d15abbd744d2db6019a79a058baa8db88ac0daf1c682ff5fe088b249e", "case_id": "education-eval-00041" }, { "domain": "education", "scenario": "加法误用乘法", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.91, "initial_state_hash": "8a1b390d15abbd744d2db6019a79a058baa8db88ac0daf1c682ff5fe088b249e", "final_state_hash": "8a1b390d15abbd744d2db6019a79a058baa8db88ac0daf1c682ff5fe088b249e", "case_id": "education-eval-00042" }, { "domain": "education", "scenario": "加法误用乘法", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.572, "initial_state_hash": "8a1b390d15abbd744d2db6019a79a058baa8db88ac0daf1c682ff5fe088b249e", "final_state_hash": "11a68159d2b4c03f857cdfd40fcb0440d64c3ec48d186409f9b80918bde1bee7", "case_id": "education-eval-00043" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 9, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.49, "initial_state_hash": "0850232f1fed4f3d234f6a259cb1750165b56125748804f1dda3f75efb4e3912", "final_state_hash": "4c391b697cd6e4f3c401c28e6ae34b2495cd79e63165f2902a8bc99effd30f14", "case_id": "education-eval-00044" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.287, "initial_state_hash": "0850232f1fed4f3d234f6a259cb1750165b56125748804f1dda3f75efb4e3912", "final_state_hash": "0850232f1fed4f3d234f6a259cb1750165b56125748804f1dda3f75efb4e3912", "case_id": "education-eval-00045" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "diagnosis_matches_evidence", "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.993, "initial_state_hash": "0850232f1fed4f3d234f6a259cb1750165b56125748804f1dda3f75efb4e3912", "final_state_hash": "0850232f1fed4f3d234f6a259cb1750165b56125748804f1dda3f75efb4e3912", "case_id": "education-eval-00046" }, { "domain": "education", "scenario": "正确作答后的迁移", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "transfer_completed", "feedback_after_each_practice" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.599, "initial_state_hash": "0850232f1fed4f3d234f6a259cb1750165b56125748804f1dda3f75efb4e3912", "final_state_hash": "9c921ddac982ce40f7122a2738d8070b98beb655c111a42aaea65829c22dfe68", "case_id": "education-eval-00047" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.304, "initial_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "final_state_hash": "dc5e6cd846b36900c814d1b2f0e1dec6b26f5e228764b82cc4dbfa88b2d2c59a", "case_id": "marketing-train-00048" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.304, "initial_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "final_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "case_id": "marketing-train-00049" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.562, "initial_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "final_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "case_id": "marketing-train-00050" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.568, "initial_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "final_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "case_id": "marketing-train-00051" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.133, "initial_state_hash": "fe65af3c3c717c4d08291380d32f375d4be0324f63fdd4068d3c9a34fec8e0b6", "final_state_hash": "b60a40a666f3783edf03383b20e3fa5b2323e05e312b3038fbbada3f03048b5c", "case_id": "marketing-train-00052" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.285, "initial_state_hash": "fe65af3c3c717c4d08291380d32f375d4be0324f63fdd4068d3c9a34fec8e0b6", "final_state_hash": "fe65af3c3c717c4d08291380d32f375d4be0324f63fdd4068d3c9a34fec8e0b6", "case_id": "marketing-train-00053" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.527, "initial_state_hash": "fe65af3c3c717c4d08291380d32f375d4be0324f63fdd4068d3c9a34fec8e0b6", "final_state_hash": "fe65af3c3c717c4d08291380d32f375d4be0324f63fdd4068d3c9a34fec8e0b6", "case_id": "marketing-train-00054" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.56, "initial_state_hash": "fe65af3c3c717c4d08291380d32f375d4be0324f63fdd4068d3c9a34fec8e0b6", "final_state_hash": "fe65af3c3c717c4d08291380d32f375d4be0324f63fdd4068d3c9a34fec8e0b6", "case_id": "marketing-train-00055" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 6, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.259, "initial_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "final_state_hash": "f4a1ff7991c3c2cf324f48a2ef17a15c87e3be20db052f7016b11fa8bb7436ee", "case_id": "marketing-train-00056" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.284, "initial_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "final_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "case_id": "marketing-train-00057" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.509, "initial_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "final_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "case_id": "marketing-train-00058" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.564, "initial_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "final_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "case_id": "marketing-train-00059" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.093, "initial_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "final_state_hash": "d5a310b287519e17b832275c0b14db0e4863be8d500e82ff65588058c9e18555", "case_id": "marketing-train-00060" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.291, "initial_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "final_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "case_id": "marketing-train-00061" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.507, "initial_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "final_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "case_id": "marketing-train-00062" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.558, "initial_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "final_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "case_id": "marketing-train-00063" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.099, "initial_state_hash": "e35b321a4efc9b9f9fb929827053ae88d18d625aceb18318477da203d3623d70", "final_state_hash": "f9307b6ca896927a966213c0c5a4d114dd06e6e18f06622e683c3b0d9c7e249f", "case_id": "marketing-train-00064" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.282, "initial_state_hash": "e35b321a4efc9b9f9fb929827053ae88d18d625aceb18318477da203d3623d70", "final_state_hash": "e35b321a4efc9b9f9fb929827053ae88d18d625aceb18318477da203d3623d70", "case_id": "marketing-train-00065" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.51, "initial_state_hash": "e35b321a4efc9b9f9fb929827053ae88d18d625aceb18318477da203d3623d70", "final_state_hash": "e35b321a4efc9b9f9fb929827053ae88d18d625aceb18318477da203d3623d70", "case_id": "marketing-train-00066" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.561, "initial_state_hash": "e35b321a4efc9b9f9fb929827053ae88d18d625aceb18318477da203d3623d70", "final_state_hash": "e35b321a4efc9b9f9fb929827053ae88d18d625aceb18318477da203d3623d70", "case_id": "marketing-train-00067" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 6, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.268, "initial_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "final_state_hash": "f4a1ff7991c3c2cf324f48a2ef17a15c87e3be20db052f7016b11fa8bb7436ee", "case_id": "marketing-train-00068" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.361, "initial_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "final_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "case_id": "marketing-train-00069" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.612, "initial_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "final_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "case_id": "marketing-train-00070" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.603, "initial_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "final_state_hash": "455e1b979972ded2a90d8d6a65749328f1f04c56500a1c27cbc346457756f046", "case_id": "marketing-train-00071" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.138, "initial_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "final_state_hash": "dc5e6cd846b36900c814d1b2f0e1dec6b26f5e228764b82cc4dbfa88b2d2c59a", "case_id": "marketing-eval-00072" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.289, "initial_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "final_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "case_id": "marketing-eval-00073" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.573, "initial_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "final_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "case_id": "marketing-eval-00074" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.57, "initial_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "final_state_hash": "a0488b4d7ab5b7903756eef34c6e2344001384761b0253a3917c1b8071828e56", "case_id": "marketing-eval-00075" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.102, "initial_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "final_state_hash": "22aec21f8d1206093ac19117ed158013ae4f16f361f04233376b91c3ab5d004b", "case_id": "marketing-eval-00076" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.293, "initial_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "final_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "case_id": "marketing-eval-00077" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.553, "initial_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "final_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "case_id": "marketing-eval-00078" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.567, "initial_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "final_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "case_id": "marketing-eval-00079" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 6, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.273, "initial_state_hash": "5d28da4f54d0e39deb1597abb7b68419a3a27a636ba32e48787155f31bc7a009", "final_state_hash": "7289475804324a4df58ad2b77199dc3b9bdaadc361f4240e0f4e469af1ddac3a", "case_id": "marketing-eval-00080" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.289, "initial_state_hash": "5d28da4f54d0e39deb1597abb7b68419a3a27a636ba32e48787155f31bc7a009", "final_state_hash": "5d28da4f54d0e39deb1597abb7b68419a3a27a636ba32e48787155f31bc7a009", "case_id": "marketing-eval-00081" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.588, "initial_state_hash": "5d28da4f54d0e39deb1597abb7b68419a3a27a636ba32e48787155f31bc7a009", "final_state_hash": "5d28da4f54d0e39deb1597abb7b68419a3a27a636ba32e48787155f31bc7a009", "case_id": "marketing-eval-00082" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.563, "initial_state_hash": "5d28da4f54d0e39deb1597abb7b68419a3a27a636ba32e48787155f31bc7a009", "final_state_hash": "5d28da4f54d0e39deb1597abb7b68419a3a27a636ba32e48787155f31bc7a009", "case_id": "marketing-eval-00083" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.086, "initial_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "final_state_hash": "d5a310b287519e17b832275c0b14db0e4863be8d500e82ff65588058c9e18555", "case_id": "marketing-eval-00084" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.292, "initial_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "final_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "case_id": "marketing-eval-00085" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.545, "initial_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "final_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "case_id": "marketing-eval-00086" }, { "domain": "marketing", "scenario": "搜索渠道效率实验", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.58, "initial_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "final_state_hash": "cf3a7dad9efefab3ed8a452abbfbc77aec34e9b6b79fd09e883b99ab86abc897", "case_id": "marketing-eval-00087" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.109, "initial_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "final_state_hash": "22aec21f8d1206093ac19117ed158013ae4f16f361f04233376b91c3ab5d004b", "case_id": "marketing-eval-00088" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.284, "initial_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "final_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "case_id": "marketing-eval-00089" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.539, "initial_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "final_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "case_id": "marketing-eval-00090" }, { "domain": "marketing", "scenario": "社区渠道效率实验", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.576, "initial_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "final_state_hash": "dc6c37ad67bf61d45bded1f8ce970aabd917030820d12482cb449e434a5fc59b", "case_id": "marketing-eval-00091" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 6, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.245, "initial_state_hash": "d23d42740d2ab5ca6f7cf988b69ee9259ab51b9578fac8e1e8ad8dff3608ebb1", "final_state_hash": "cd0da5830fbbf8805927d52c7df8e84d5ceb6fd2e2c7bdd42150d75401fb8800", "case_id": "marketing-eval-00092" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.287, "initial_state_hash": "d23d42740d2ab5ca6f7cf988b69ee9259ab51b9578fac8e1e8ad8dff3608ebb1", "final_state_hash": "d23d42740d2ab5ca6f7cf988b69ee9259ab51b9578fac8e1e8ad8dff3608ebb1", "case_id": "marketing-eval-00093" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 18, "terminated": false, "truncated": true, "invalid_actions": 18, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.542, "initial_state_hash": "d23d42740d2ab5ca6f7cf988b69ee9259ab51b9578fac8e1e8ad8dff3608ebb1", "final_state_hash": "d23d42740d2ab5ca6f7cf988b69ee9259ab51b9578fac8e1e8ad8dff3608ebb1", "case_id": "marketing-eval-00094" }, { "domain": "marketing", "scenario": "分散渠道预算约束", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "budget_conserved", "claims_grounded", "current_experiment_cited", "qualified_target_met", "channel_constraints_met" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.568, "initial_state_hash": "d23d42740d2ab5ca6f7cf988b69ee9259ab51b9578fac8e1e8ad8dff3608ebb1", "final_state_hash": "d23d42740d2ab5ca6f7cf988b69ee9259ab51b9578fac8e1e8ad8dff3608ebb1", "case_id": "marketing-eval-00095" }, { "domain": "office", "scenario": "顺序上线", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 3.297, "initial_state_hash": "8ba344f1c4223190d8e5d9a2fa781315157c99350c407bc334e3fbd62e2dd366", "final_state_hash": "51e05f218fe83aac3569aadf39a036f335d615ebc54c6562fc83d21878833fdd", "case_id": "office-train-00096" }, { "domain": "office", "scenario": "顺序上线", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.298, "initial_state_hash": "8ba344f1c4223190d8e5d9a2fa781315157c99350c407bc334e3fbd62e2dd366", "final_state_hash": "8ba344f1c4223190d8e5d9a2fa781315157c99350c407bc334e3fbd62e2dd366", "case_id": "office-train-00097" }, { "domain": "office", "scenario": "顺序上线", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.896, "initial_state_hash": "8ba344f1c4223190d8e5d9a2fa781315157c99350c407bc334e3fbd62e2dd366", "final_state_hash": "8ba344f1c4223190d8e5d9a2fa781315157c99350c407bc334e3fbd62e2dd366", "case_id": "office-train-00098" }, { "domain": "office", "scenario": "顺序上线", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.835, "initial_state_hash": "8ba344f1c4223190d8e5d9a2fa781315157c99350c407bc334e3fbd62e2dd366", "final_state_hash": "6d6428b840366412ca1b8be5206744732866c925f4c89700e0357feef278c48d", "case_id": "office-train-00099" }, { "domain": "office", "scenario": "并行评审与检查", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.408, "initial_state_hash": "cbb733c80fbd0c4b86047980deac648fcde16d0f366e38b802db0971e9d44fb4", "final_state_hash": "c5954054aa43fbbb3609c126c9b951db5f0cd09ffc81762683823950102bc532", "case_id": "office-train-00100" }, { "domain": "office", "scenario": "并行评审与检查", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.301, "initial_state_hash": "cbb733c80fbd0c4b86047980deac648fcde16d0f366e38b802db0971e9d44fb4", "final_state_hash": "cbb733c80fbd0c4b86047980deac648fcde16d0f366e38b802db0971e9d44fb4", "case_id": "office-train-00101" }, { "domain": "office", "scenario": "并行评审与检查", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.873, "initial_state_hash": "cbb733c80fbd0c4b86047980deac648fcde16d0f366e38b802db0971e9d44fb4", "final_state_hash": "cbb733c80fbd0c4b86047980deac648fcde16d0f366e38b802db0971e9d44fb4", "case_id": "office-train-00102" }, { "domain": "office", "scenario": "并行评审与检查", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.844, "initial_state_hash": "cbb733c80fbd0c4b86047980deac648fcde16d0f366e38b802db0971e9d44fb4", "final_state_hash": "ad6e515199a6004feec6a78ec06ba1646ac6dbf6053dd62b2eb8e99c75b1c6d3", "case_id": "office-train-00103" }, { "domain": "office", "scenario": "先检查后评审", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.811, "initial_state_hash": "1087334a033f8a3c75fa5f30654d31adadf0de527ca5d9e37f69fdd71dd0b264", "final_state_hash": "6b0c4b967ac552e4362469564a8c092777168c0462a8249791de31e07d8fe8fb", "case_id": "office-train-00104" }, { "domain": "office", "scenario": "先检查后评审", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.296, "initial_state_hash": "1087334a033f8a3c75fa5f30654d31adadf0de527ca5d9e37f69fdd71dd0b264", "final_state_hash": "1087334a033f8a3c75fa5f30654d31adadf0de527ca5d9e37f69fdd71dd0b264", "case_id": "office-train-00105" }, { "domain": "office", "scenario": "先检查后评审", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.889, "initial_state_hash": "1087334a033f8a3c75fa5f30654d31adadf0de527ca5d9e37f69fdd71dd0b264", "final_state_hash": "1087334a033f8a3c75fa5f30654d31adadf0de527ca5d9e37f69fdd71dd0b264", "case_id": "office-train-00106" }, { "domain": "office", "scenario": "先检查后评审", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.824, "initial_state_hash": "1087334a033f8a3c75fa5f30654d31adadf0de527ca5d9e37f69fdd71dd0b264", "final_state_hash": "c3e79277d2f3a014b77e816ed4fad7e3de65864293d9e86d3d5c5117bc8a550c", "case_id": "office-train-00107" }, { "domain": "office", "scenario": "顺序上线", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.378, "initial_state_hash": "12e0486c7099ec838d17ec2bfed76f3acf2111a65fc4fb7dc7ba696fa23d4104", "final_state_hash": "b0e304c8dfde798a7f8ade7e4e081b26f786d3cedc526f3959d970119faaa511", "case_id": "office-train-00108" }, { "domain": "office", "scenario": "顺序上线", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.288, "initial_state_hash": "12e0486c7099ec838d17ec2bfed76f3acf2111a65fc4fb7dc7ba696fa23d4104", "final_state_hash": "12e0486c7099ec838d17ec2bfed76f3acf2111a65fc4fb7dc7ba696fa23d4104", "case_id": "office-train-00109" }, { "domain": "office", "scenario": "顺序上线", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.848, "initial_state_hash": "12e0486c7099ec838d17ec2bfed76f3acf2111a65fc4fb7dc7ba696fa23d4104", "final_state_hash": "12e0486c7099ec838d17ec2bfed76f3acf2111a65fc4fb7dc7ba696fa23d4104", "case_id": "office-train-00110" }, { "domain": "office", "scenario": "顺序上线", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.828, "initial_state_hash": "12e0486c7099ec838d17ec2bfed76f3acf2111a65fc4fb7dc7ba696fa23d4104", "final_state_hash": "a2bc6d5d50ee638798bf40f5ba52c6a1a4b46ecbf429995da4ec736a6697a3de", "case_id": "office-train-00111" }, { "domain": "office", "scenario": "并行评审与检查", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.363, "initial_state_hash": "b5f690a5a33d39b3f8d2a5645eecac0a24b83d3ebfe457e0d426ec1d3bde2d41", "final_state_hash": "4efa21570dedcd683a55aac8aacf3acfabda8659b5080a3adbbe2c79a0aa484a", "case_id": "office-train-00112" }, { "domain": "office", "scenario": "并行评审与检查", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.298, "initial_state_hash": "b5f690a5a33d39b3f8d2a5645eecac0a24b83d3ebfe457e0d426ec1d3bde2d41", "final_state_hash": "b5f690a5a33d39b3f8d2a5645eecac0a24b83d3ebfe457e0d426ec1d3bde2d41", "case_id": "office-train-00113" }, { "domain": "office", "scenario": "并行评审与检查", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.829, "initial_state_hash": "b5f690a5a33d39b3f8d2a5645eecac0a24b83d3ebfe457e0d426ec1d3bde2d41", "final_state_hash": "b5f690a5a33d39b3f8d2a5645eecac0a24b83d3ebfe457e0d426ec1d3bde2d41", "case_id": "office-train-00114" }, { "domain": "office", "scenario": "并行评审与检查", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.824, "initial_state_hash": "b5f690a5a33d39b3f8d2a5645eecac0a24b83d3ebfe457e0d426ec1d3bde2d41", "final_state_hash": "9570da914a3c9b3912dfb1fbe93cc32059f9923e40a37a53f7119677f780e72d", "case_id": "office-train-00115" }, { "domain": "office", "scenario": "先检查后评审", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.372, "initial_state_hash": "f1c9ed5d4a964c97f7e1d66b86480122edb2189a4e6445c245cf9b1eff82daa5", "final_state_hash": "cf5a8642f55ac2c5a58075e47050dfa3785ebe1a0091af99064e06fd04972fde", "case_id": "office-train-00116" }, { "domain": "office", "scenario": "先检查后评审", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.286, "initial_state_hash": "f1c9ed5d4a964c97f7e1d66b86480122edb2189a4e6445c245cf9b1eff82daa5", "final_state_hash": "f1c9ed5d4a964c97f7e1d66b86480122edb2189a4e6445c245cf9b1eff82daa5", "case_id": "office-train-00117" }, { "domain": "office", "scenario": "先检查后评审", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.836, "initial_state_hash": "f1c9ed5d4a964c97f7e1d66b86480122edb2189a4e6445c245cf9b1eff82daa5", "final_state_hash": "f1c9ed5d4a964c97f7e1d66b86480122edb2189a4e6445c245cf9b1eff82daa5", "case_id": "office-train-00118" }, { "domain": "office", "scenario": "先检查后评审", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.83, "initial_state_hash": "f1c9ed5d4a964c97f7e1d66b86480122edb2189a4e6445c245cf9b1eff82daa5", "final_state_hash": "a67363024b46e468b17e16b9b93905b88e1408b32c0bcf646dc45d219cebc971", "case_id": "office-train-00119" }, { "domain": "office", "scenario": "顺序上线", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.347, "initial_state_hash": "ec39194f49b7ce23bf8e53d85d209542b63c10bbc4214ac03b2d1ce41f6d1ccd", "final_state_hash": "4cb827021321d9c313abf8c857900cd0160d91c926510239a0f4d46719369d47", "case_id": "office-eval-00120" }, { "domain": "office", "scenario": "顺序上线", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.288, "initial_state_hash": "ec39194f49b7ce23bf8e53d85d209542b63c10bbc4214ac03b2d1ce41f6d1ccd", "final_state_hash": "ec39194f49b7ce23bf8e53d85d209542b63c10bbc4214ac03b2d1ce41f6d1ccd", "case_id": "office-eval-00121" }, { "domain": "office", "scenario": "顺序上线", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.851, "initial_state_hash": "ec39194f49b7ce23bf8e53d85d209542b63c10bbc4214ac03b2d1ce41f6d1ccd", "final_state_hash": "ec39194f49b7ce23bf8e53d85d209542b63c10bbc4214ac03b2d1ce41f6d1ccd", "case_id": "office-eval-00122" }, { "domain": "office", "scenario": "顺序上线", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.817, "initial_state_hash": "ec39194f49b7ce23bf8e53d85d209542b63c10bbc4214ac03b2d1ce41f6d1ccd", "final_state_hash": "3b9589218707131e49b2a4aa2ce7b538fe941878bbe4d99460333a813efe69f2", "case_id": "office-eval-00123" }, { "domain": "office", "scenario": "并行评审与检查", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.349, "initial_state_hash": "b8328c58618ac9191e5474e146069c08742c1f0d8634ced322acf914389fe977", "final_state_hash": "60b1e5d22369a62ba4ea0af04e20b830d4058e6af051d21933f35ca831ad048e", "case_id": "office-eval-00124" }, { "domain": "office", "scenario": "并行评审与检查", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.284, "initial_state_hash": "b8328c58618ac9191e5474e146069c08742c1f0d8634ced322acf914389fe977", "final_state_hash": "b8328c58618ac9191e5474e146069c08742c1f0d8634ced322acf914389fe977", "case_id": "office-eval-00125" }, { "domain": "office", "scenario": "并行评审与检查", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.854, "initial_state_hash": "b8328c58618ac9191e5474e146069c08742c1f0d8634ced322acf914389fe977", "final_state_hash": "b8328c58618ac9191e5474e146069c08742c1f0d8634ced322acf914389fe977", "case_id": "office-eval-00126" }, { "domain": "office", "scenario": "并行评审与检查", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.818, "initial_state_hash": "b8328c58618ac9191e5474e146069c08742c1f0d8634ced322acf914389fe977", "final_state_hash": "074f06c7ae6231a654ab7aba30109a6f98373d5395e81510dee7b8b97266630a", "case_id": "office-eval-00127" }, { "domain": "office", "scenario": "先检查后评审", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.355, "initial_state_hash": "2fa19e6c66385ed351cc0b9dc4ed3b4cb601895d5e6340eed8352c8b317059eb", "final_state_hash": "5c5ef82919390efb30b623d7f63cd21facc3befb1ba9f93e6d9b2e2eac40a97f", "case_id": "office-eval-00128" }, { "domain": "office", "scenario": "先检查后评审", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.288, "initial_state_hash": "2fa19e6c66385ed351cc0b9dc4ed3b4cb601895d5e6340eed8352c8b317059eb", "final_state_hash": "2fa19e6c66385ed351cc0b9dc4ed3b4cb601895d5e6340eed8352c8b317059eb", "case_id": "office-eval-00129" }, { "domain": "office", "scenario": "先检查后评审", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.851, "initial_state_hash": "2fa19e6c66385ed351cc0b9dc4ed3b4cb601895d5e6340eed8352c8b317059eb", "final_state_hash": "2fa19e6c66385ed351cc0b9dc4ed3b4cb601895d5e6340eed8352c8b317059eb", "case_id": "office-eval-00130" }, { "domain": "office", "scenario": "先检查后评审", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.825, "initial_state_hash": "2fa19e6c66385ed351cc0b9dc4ed3b4cb601895d5e6340eed8352c8b317059eb", "final_state_hash": "e8558718f5cab77caed1994a828eec887682a05cbd53b9d96f696ceafb4e9c81", "case_id": "office-eval-00131" }, { "domain": "office", "scenario": "顺序上线", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.372, "initial_state_hash": "0db2a387ab0ae3cd6be712ffa94cc7730ce8fedba2ca1090c561112683b8ca48", "final_state_hash": "9464531b19647512bed75b11b7cb8322417c048be1a72cc0dfa889f2fcebd91b", "case_id": "office-eval-00132" }, { "domain": "office", "scenario": "顺序上线", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.285, "initial_state_hash": "0db2a387ab0ae3cd6be712ffa94cc7730ce8fedba2ca1090c561112683b8ca48", "final_state_hash": "0db2a387ab0ae3cd6be712ffa94cc7730ce8fedba2ca1090c561112683b8ca48", "case_id": "office-eval-00133" }, { "domain": "office", "scenario": "顺序上线", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.829, "initial_state_hash": "0db2a387ab0ae3cd6be712ffa94cc7730ce8fedba2ca1090c561112683b8ca48", "final_state_hash": "0db2a387ab0ae3cd6be712ffa94cc7730ce8fedba2ca1090c561112683b8ca48", "case_id": "office-eval-00134" }, { "domain": "office", "scenario": "顺序上线", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.83, "initial_state_hash": "0db2a387ab0ae3cd6be712ffa94cc7730ce8fedba2ca1090c561112683b8ca48", "final_state_hash": "ed270fc8f72b4db1371933a6291837f67904496c755f434e353d33cfd7e26336", "case_id": "office-eval-00135" }, { "domain": "office", "scenario": "并行评审与检查", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.341, "initial_state_hash": "f5a78a4939b2d32af60585189e791f21c521aebec421387d969b38c0b2405376", "final_state_hash": "fccfd4455ece804f6fa680aca858580f361d4d03eccd02a5a50ec2f9a27a5255", "case_id": "office-eval-00136" }, { "domain": "office", "scenario": "并行评审与检查", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.287, "initial_state_hash": "f5a78a4939b2d32af60585189e791f21c521aebec421387d969b38c0b2405376", "final_state_hash": "f5a78a4939b2d32af60585189e791f21c521aebec421387d969b38c0b2405376", "case_id": "office-eval-00137" }, { "domain": "office", "scenario": "并行评审与检查", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.962, "initial_state_hash": "f5a78a4939b2d32af60585189e791f21c521aebec421387d969b38c0b2405376", "final_state_hash": "f5a78a4939b2d32af60585189e791f21c521aebec421387d969b38c0b2405376", "case_id": "office-eval-00138" }, { "domain": "office", "scenario": "并行评审与检查", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.844, "initial_state_hash": "f5a78a4939b2d32af60585189e791f21c521aebec421387d969b38c0b2405376", "final_state_hash": "4e3ab61c36141813df367c0fdc37cb99b7686a271e7e50a05ad04be5bb633e3a", "case_id": "office-eval-00139" }, { "domain": "office", "scenario": "先检查后评审", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 11, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.407, "initial_state_hash": "9975784861a473afa4a71161fcbcf76dc790d667a8a85575bccf2e64b2ade4ed", "final_state_hash": "f576df38f2df6b51102fd6f9090f7589f56675f75659014b1f0cc05d537329da", "case_id": "office-eval-00140" }, { "domain": "office", "scenario": "先检查后评审", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.29, "initial_state_hash": "9975784861a473afa4a71161fcbcf76dc790d667a8a85575bccf2e64b2ade4ed", "final_state_hash": "9975784861a473afa4a71161fcbcf76dc790d667a8a85575bccf2e64b2ade4ed", "case_id": "office-eval-00141" }, { "domain": "office", "scenario": "先检查后评审", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 20, "terminated": false, "truncated": true, "invalid_actions": 20, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 2.819, "initial_state_hash": "9975784861a473afa4a71161fcbcf76dc790d667a8a85575bccf2e64b2ade4ed", "final_state_hash": "9975784861a473afa4a71161fcbcf76dc790d667a8a85575bccf2e64b2ade4ed", "case_id": "office-eval-00142" }, { "domain": "office", "scenario": "先检查后评审", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 1, "failed_checks": [ "all_deliverables_complete", "handoff_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.821, "initial_state_hash": "9975784861a473afa4a71161fcbcf76dc790d667a8a85575bccf2e64b2ade4ed", "final_state_hash": "9a89917d325369259a03fc08ea232594b0eb1ef822e072a64537eb9fe0901705", "case_id": "office-eval-00143" }, { "domain": "math", "scenario": "整数解方程", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.867, "initial_state_hash": "9781d2c8e2ae6890a4eda10a67bcb59c015add6d8759cb5dcadb9241fbfa1286", "final_state_hash": "0f20e3bf2d7fd53c4cad012db77879f4a590f2b4cdffb0df4a54d751cf19926e", "case_id": "math-train-00144" }, { "domain": "math", "scenario": "整数解方程", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.223, "initial_state_hash": "9781d2c8e2ae6890a4eda10a67bcb59c015add6d8759cb5dcadb9241fbfa1286", "final_state_hash": "9781d2c8e2ae6890a4eda10a67bcb59c015add6d8759cb5dcadb9241fbfa1286", "case_id": "math-train-00145" }, { "domain": "math", "scenario": "整数解方程", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.725, "initial_state_hash": "9781d2c8e2ae6890a4eda10a67bcb59c015add6d8759cb5dcadb9241fbfa1286", "final_state_hash": "9781d2c8e2ae6890a4eda10a67bcb59c015add6d8759cb5dcadb9241fbfa1286", "case_id": "math-train-00146" }, { "domain": "math", "scenario": "整数解方程", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.48, "initial_state_hash": "9781d2c8e2ae6890a4eda10a67bcb59c015add6d8759cb5dcadb9241fbfa1286", "final_state_hash": "9781d2c8e2ae6890a4eda10a67bcb59c015add6d8759cb5dcadb9241fbfa1286", "case_id": "math-train-00147" }, { "domain": "math", "scenario": "分数解方程", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.86, "initial_state_hash": "28da92131d77f231b9dc0e0951de77974014d14a7f1310dda1d5922079fd0483", "final_state_hash": "48ff88e059a5c4b03bd17b3596c6556e45d6175e793abd9c7c2a62754c02d3c5", "case_id": "math-train-00148" }, { "domain": "math", "scenario": "分数解方程", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.215, "initial_state_hash": "28da92131d77f231b9dc0e0951de77974014d14a7f1310dda1d5922079fd0483", "final_state_hash": "28da92131d77f231b9dc0e0951de77974014d14a7f1310dda1d5922079fd0483", "case_id": "math-train-00149" }, { "domain": "math", "scenario": "分数解方程", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.701, "initial_state_hash": "28da92131d77f231b9dc0e0951de77974014d14a7f1310dda1d5922079fd0483", "final_state_hash": "28da92131d77f231b9dc0e0951de77974014d14a7f1310dda1d5922079fd0483", "case_id": "math-train-00150" }, { "domain": "math", "scenario": "分数解方程", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.477, "initial_state_hash": "28da92131d77f231b9dc0e0951de77974014d14a7f1310dda1d5922079fd0483", "final_state_hash": "28da92131d77f231b9dc0e0951de77974014d14a7f1310dda1d5922079fd0483", "case_id": "math-train-00151" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.827, "initial_state_hash": "f04986df52b1581ceb87caacd8dffd84a1595bdc461a5b80b0e48c4e453fe0b2", "final_state_hash": "d05c332968832b9676deb3450b7af49857acc968a00db0c3163887e05637bf05", "case_id": "math-train-00152" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.21, "initial_state_hash": "f04986df52b1581ceb87caacd8dffd84a1595bdc461a5b80b0e48c4e453fe0b2", "final_state_hash": "f04986df52b1581ceb87caacd8dffd84a1595bdc461a5b80b0e48c4e453fe0b2", "case_id": "math-train-00153" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.689, "initial_state_hash": "f04986df52b1581ceb87caacd8dffd84a1595bdc461a5b80b0e48c4e453fe0b2", "final_state_hash": "f04986df52b1581ceb87caacd8dffd84a1595bdc461a5b80b0e48c4e453fe0b2", "case_id": "math-train-00154" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.472, "initial_state_hash": "f04986df52b1581ceb87caacd8dffd84a1595bdc461a5b80b0e48c4e453fe0b2", "final_state_hash": "f04986df52b1581ceb87caacd8dffd84a1595bdc461a5b80b0e48c4e453fe0b2", "case_id": "math-train-00155" }, { "domain": "math", "scenario": "整数解方程", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.824, "initial_state_hash": "e8a0c54783c34bfd38c0c721ad0a75de3eb2b18a0800ff0af7047d8580db2929", "final_state_hash": "5fffc16719a45e0a503491afe231d701ccdb0b9cd0f74aff44449373095ef702", "case_id": "math-train-00156" }, { "domain": "math", "scenario": "整数解方程", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.211, "initial_state_hash": "e8a0c54783c34bfd38c0c721ad0a75de3eb2b18a0800ff0af7047d8580db2929", "final_state_hash": "e8a0c54783c34bfd38c0c721ad0a75de3eb2b18a0800ff0af7047d8580db2929", "case_id": "math-train-00157" }, { "domain": "math", "scenario": "整数解方程", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.696, "initial_state_hash": "e8a0c54783c34bfd38c0c721ad0a75de3eb2b18a0800ff0af7047d8580db2929", "final_state_hash": "e8a0c54783c34bfd38c0c721ad0a75de3eb2b18a0800ff0af7047d8580db2929", "case_id": "math-train-00158" }, { "domain": "math", "scenario": "整数解方程", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.491, "initial_state_hash": "e8a0c54783c34bfd38c0c721ad0a75de3eb2b18a0800ff0af7047d8580db2929", "final_state_hash": "e8a0c54783c34bfd38c0c721ad0a75de3eb2b18a0800ff0af7047d8580db2929", "case_id": "math-train-00159" }, { "domain": "math", "scenario": "分数解方程", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.84, "initial_state_hash": "a572245dcc7f44cc346e8962ee08f16a49b304efac5b5acad0fecba5caf0e238", "final_state_hash": "69e4c3c0e2c345cfea5da519decea7571a3eca1d8ae939c813219e2e697301d1", "case_id": "math-train-00160" }, { "domain": "math", "scenario": "分数解方程", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.217, "initial_state_hash": "a572245dcc7f44cc346e8962ee08f16a49b304efac5b5acad0fecba5caf0e238", "final_state_hash": "a572245dcc7f44cc346e8962ee08f16a49b304efac5b5acad0fecba5caf0e238", "case_id": "math-train-00161" }, { "domain": "math", "scenario": "分数解方程", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.712, "initial_state_hash": "a572245dcc7f44cc346e8962ee08f16a49b304efac5b5acad0fecba5caf0e238", "final_state_hash": "a572245dcc7f44cc346e8962ee08f16a49b304efac5b5acad0fecba5caf0e238", "case_id": "math-train-00162" }, { "domain": "math", "scenario": "分数解方程", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.479, "initial_state_hash": "a572245dcc7f44cc346e8962ee08f16a49b304efac5b5acad0fecba5caf0e238", "final_state_hash": "a572245dcc7f44cc346e8962ee08f16a49b304efac5b5acad0fecba5caf0e238", "case_id": "math-train-00163" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.824, "initial_state_hash": "fa56a02e8fde2c75d5fb3346689369598676aa0499be5a08b3b713636843dbc2", "final_state_hash": "73d467fa1ea6b8a1509254cde112bab1f73daed46fdeb50bf012ea1bd6142bb5", "case_id": "math-train-00164" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.213, "initial_state_hash": "fa56a02e8fde2c75d5fb3346689369598676aa0499be5a08b3b713636843dbc2", "final_state_hash": "fa56a02e8fde2c75d5fb3346689369598676aa0499be5a08b3b713636843dbc2", "case_id": "math-train-00165" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.708, "initial_state_hash": "fa56a02e8fde2c75d5fb3346689369598676aa0499be5a08b3b713636843dbc2", "final_state_hash": "fa56a02e8fde2c75d5fb3346689369598676aa0499be5a08b3b713636843dbc2", "case_id": "math-train-00166" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.471, "initial_state_hash": "fa56a02e8fde2c75d5fb3346689369598676aa0499be5a08b3b713636843dbc2", "final_state_hash": "fa56a02e8fde2c75d5fb3346689369598676aa0499be5a08b3b713636843dbc2", "case_id": "math-train-00167" }, { "domain": "math", "scenario": "整数解方程", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.835, "initial_state_hash": "1ce0c1352586ab530c930f2e7bc3c155c29523b3efe2ac8034f3cb3ea3309da0", "final_state_hash": "d8372dbb828413a5665b8c09b7042f739cfeba5b9701e55f3d37772a4c318698", "case_id": "math-eval-00168" }, { "domain": "math", "scenario": "整数解方程", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.215, "initial_state_hash": "1ce0c1352586ab530c930f2e7bc3c155c29523b3efe2ac8034f3cb3ea3309da0", "final_state_hash": "1ce0c1352586ab530c930f2e7bc3c155c29523b3efe2ac8034f3cb3ea3309da0", "case_id": "math-eval-00169" }, { "domain": "math", "scenario": "整数解方程", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.69, "initial_state_hash": "1ce0c1352586ab530c930f2e7bc3c155c29523b3efe2ac8034f3cb3ea3309da0", "final_state_hash": "1ce0c1352586ab530c930f2e7bc3c155c29523b3efe2ac8034f3cb3ea3309da0", "case_id": "math-eval-00170" }, { "domain": "math", "scenario": "整数解方程", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.469, "initial_state_hash": "1ce0c1352586ab530c930f2e7bc3c155c29523b3efe2ac8034f3cb3ea3309da0", "final_state_hash": "1ce0c1352586ab530c930f2e7bc3c155c29523b3efe2ac8034f3cb3ea3309da0", "case_id": "math-eval-00171" }, { "domain": "math", "scenario": "分数解方程", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.826, "initial_state_hash": "b9e750e169bbdb9fad42fc8017e3d4c03d42ac906e09c51b3999a5c24320195f", "final_state_hash": "1f306f9a414618510893aaee9ebb3a3927bccc4226f69c0d76d8e7c3b683c1b6", "case_id": "math-eval-00172" }, { "domain": "math", "scenario": "分数解方程", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.214, "initial_state_hash": "b9e750e169bbdb9fad42fc8017e3d4c03d42ac906e09c51b3999a5c24320195f", "final_state_hash": "b9e750e169bbdb9fad42fc8017e3d4c03d42ac906e09c51b3999a5c24320195f", "case_id": "math-eval-00173" }, { "domain": "math", "scenario": "分数解方程", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.722, "initial_state_hash": "b9e750e169bbdb9fad42fc8017e3d4c03d42ac906e09c51b3999a5c24320195f", "final_state_hash": "b9e750e169bbdb9fad42fc8017e3d4c03d42ac906e09c51b3999a5c24320195f", "case_id": "math-eval-00174" }, { "domain": "math", "scenario": "分数解方程", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.475, "initial_state_hash": "b9e750e169bbdb9fad42fc8017e3d4c03d42ac906e09c51b3999a5c24320195f", "final_state_hash": "b9e750e169bbdb9fad42fc8017e3d4c03d42ac906e09c51b3999a5c24320195f", "case_id": "math-eval-00175" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.818, "initial_state_hash": "f11e35d6dff9ad200ccdc40cbf0d904139988fd76ad4253e010e55e621acad02", "final_state_hash": "14c449ce8586ecf9be3cc32b4ee4c9994f6b0bbb6524d4e6390883083a145efc", "case_id": "math-eval-00176" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.213, "initial_state_hash": "f11e35d6dff9ad200ccdc40cbf0d904139988fd76ad4253e010e55e621acad02", "final_state_hash": "f11e35d6dff9ad200ccdc40cbf0d904139988fd76ad4253e010e55e621acad02", "case_id": "math-eval-00177" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.714, "initial_state_hash": "f11e35d6dff9ad200ccdc40cbf0d904139988fd76ad4253e010e55e621acad02", "final_state_hash": "f11e35d6dff9ad200ccdc40cbf0d904139988fd76ad4253e010e55e621acad02", "case_id": "math-eval-00178" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.474, "initial_state_hash": "f11e35d6dff9ad200ccdc40cbf0d904139988fd76ad4253e010e55e621acad02", "final_state_hash": "f11e35d6dff9ad200ccdc40cbf0d904139988fd76ad4253e010e55e621acad02", "case_id": "math-eval-00179" }, { "domain": "math", "scenario": "整数解方程", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.824, "initial_state_hash": "a395c19c944f8041967b628275e23c1ffee83991bd79bb1975ba996911f2ad29", "final_state_hash": "456a3ff6630dde00df3c9904fb390caceafcb73e0964333bace7abc1f84ccf78", "case_id": "math-eval-00180" }, { "domain": "math", "scenario": "整数解方程", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.211, "initial_state_hash": "a395c19c944f8041967b628275e23c1ffee83991bd79bb1975ba996911f2ad29", "final_state_hash": "a395c19c944f8041967b628275e23c1ffee83991bd79bb1975ba996911f2ad29", "case_id": "math-eval-00181" }, { "domain": "math", "scenario": "整数解方程", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.684, "initial_state_hash": "a395c19c944f8041967b628275e23c1ffee83991bd79bb1975ba996911f2ad29", "final_state_hash": "a395c19c944f8041967b628275e23c1ffee83991bd79bb1975ba996911f2ad29", "case_id": "math-eval-00182" }, { "domain": "math", "scenario": "整数解方程", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.473, "initial_state_hash": "a395c19c944f8041967b628275e23c1ffee83991bd79bb1975ba996911f2ad29", "final_state_hash": "a395c19c944f8041967b628275e23c1ffee83991bd79bb1975ba996911f2ad29", "case_id": "math-eval-00183" }, { "domain": "math", "scenario": "分数解方程", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.818, "initial_state_hash": "e80cf3524f31dccec2179b512ea7cfdb0a84d5c0efc8b11ae41b08713f12681a", "final_state_hash": "6c1a1a8f267fac6d11f6dc216c5998c25e14c04af42164a147d6791f7bd93da0", "case_id": "math-eval-00184" }, { "domain": "math", "scenario": "分数解方程", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.212, "initial_state_hash": "e80cf3524f31dccec2179b512ea7cfdb0a84d5c0efc8b11ae41b08713f12681a", "final_state_hash": "e80cf3524f31dccec2179b512ea7cfdb0a84d5c0efc8b11ae41b08713f12681a", "case_id": "math-eval-00185" }, { "domain": "math", "scenario": "分数解方程", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.692, "initial_state_hash": "e80cf3524f31dccec2179b512ea7cfdb0a84d5c0efc8b11ae41b08713f12681a", "final_state_hash": "e80cf3524f31dccec2179b512ea7cfdb0a84d5c0efc8b11ae41b08713f12681a", "case_id": "math-eval-00186" }, { "domain": "math", "scenario": "分数解方程", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.471, "initial_state_hash": "e80cf3524f31dccec2179b512ea7cfdb0a84d5c0efc8b11ae41b08713f12681a", "final_state_hash": "e80cf3524f31dccec2179b512ea7cfdb0a84d5c0efc8b11ae41b08713f12681a", "case_id": "math-eval-00187" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.824, "initial_state_hash": "c21dcdad52dbceaaff20dd0b395257f77d62b87b7726f0c78a34c42338b298f7", "final_state_hash": "ec281bc3d8c78aa5467dc003c8eda0c6e8794500091653ace3373662ffcf89ac", "case_id": "math-eval-00188" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.21, "initial_state_hash": "c21dcdad52dbceaaff20dd0b395257f77d62b87b7726f0c78a34c42338b298f7", "final_state_hash": "c21dcdad52dbceaaff20dd0b395257f77d62b87b7726f0c78a34c42338b298f7", "case_id": "math-eval-00189" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.684, "initial_state_hash": "c21dcdad52dbceaaff20dd0b395257f77d62b87b7726f0c78a34c42338b298f7", "final_state_hash": "c21dcdad52dbceaaff20dd0b395257f77d62b87b7726f0c78a34c42338b298f7", "case_id": "math-eval-00190" }, { "domain": "math", "scenario": "带负系数的精确消元", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "equivalent_elimination", "exact_solution", "derivation_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.471, "initial_state_hash": "c21dcdad52dbceaaff20dd0b395257f77d62b87b7726f0c78a34c42338b298f7", "final_state_hash": "c21dcdad52dbceaaff20dd0b395257f77d62b87b7726f0c78a34c42338b298f7", "case_id": "math-eval-00191" }, { "domain": "programming", "scenario": "首条订单去重", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 402.054, "initial_state_hash": "5f76c39cb2acc3bb45feb868599b7e98bc6b20d71a23e62122324a9a50b5389e", "final_state_hash": "cf2f83dd28c68e89f6653f48e6c294ddb5f348813c9b1c38245ed961b943194a", "case_id": "programming-train-00192" }, { "domain": "programming", "scenario": "首条订单去重", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 139.939, "initial_state_hash": "5f76c39cb2acc3bb45feb868599b7e98bc6b20d71a23e62122324a9a50b5389e", "final_state_hash": "5f76c39cb2acc3bb45feb868599b7e98bc6b20d71a23e62122324a9a50b5389e", "case_id": "programming-train-00193" }, { "domain": "programming", "scenario": "首条订单去重", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 138.109, "initial_state_hash": "5f76c39cb2acc3bb45feb868599b7e98bc6b20d71a23e62122324a9a50b5389e", "final_state_hash": "5f76c39cb2acc3bb45feb868599b7e98bc6b20d71a23e62122324a9a50b5389e", "case_id": "programming-train-00194" }, { "domain": "programming", "scenario": "首条订单去重", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 230.557, "initial_state_hash": "5f76c39cb2acc3bb45feb868599b7e98bc6b20d71a23e62122324a9a50b5389e", "final_state_hash": "345bcf5117ffe8736daf6e7f44b3ba7271ab364e07314fe1267c4e3cd9e3d548", "case_id": "programming-train-00195" }, { "domain": "programming", "scenario": "最新订单去重", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 301.833, "initial_state_hash": "ae813ee6f9bc7c70355b849c6918cf3b85e4c332d2f63cd7a599278dfd4a9f1a", "final_state_hash": "37289b8476dc4ab9e2eadca7f4bd4e515aa81b182946cbf9a92250bbbfaa53d7", "case_id": "programming-train-00196" }, { "domain": "programming", "scenario": "最新订单去重", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 140.544, "initial_state_hash": "ae813ee6f9bc7c70355b849c6918cf3b85e4c332d2f63cd7a599278dfd4a9f1a", "final_state_hash": "ae813ee6f9bc7c70355b849c6918cf3b85e4c332d2f63cd7a599278dfd4a9f1a", "case_id": "programming-train-00197" }, { "domain": "programming", "scenario": "最新订单去重", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 142.309, "initial_state_hash": "ae813ee6f9bc7c70355b849c6918cf3b85e4c332d2f63cd7a599278dfd4a9f1a", "final_state_hash": "ae813ee6f9bc7c70355b849c6918cf3b85e4c332d2f63cd7a599278dfd4a9f1a", "case_id": "programming-train-00198" }, { "domain": "programming", "scenario": "最新订单去重", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 226.234, "initial_state_hash": "ae813ee6f9bc7c70355b849c6918cf3b85e4c332d2f63cd7a599278dfd4a9f1a", "final_state_hash": "79440e0845865df15ecbee119489844b2d325a49c6faa9a186f42871e9b0f274", "case_id": "programming-train-00199" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 271.184, "initial_state_hash": "b21e43a5e73fb0cae9d3f09d0227fe2850be05e8dab551891667c22a806996e1", "final_state_hash": "cc02c97ccf17b15fe7907fd270ef4b2d17d16238549f4b294a272268033bb8ae", "case_id": "programming-train-00200" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 141.654, "initial_state_hash": "b21e43a5e73fb0cae9d3f09d0227fe2850be05e8dab551891667c22a806996e1", "final_state_hash": "b21e43a5e73fb0cae9d3f09d0227fe2850be05e8dab551891667c22a806996e1", "case_id": "programming-train-00201" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 141.605, "initial_state_hash": "b21e43a5e73fb0cae9d3f09d0227fe2850be05e8dab551891667c22a806996e1", "final_state_hash": "b21e43a5e73fb0cae9d3f09d0227fe2850be05e8dab551891667c22a806996e1", "case_id": "programming-train-00202" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 226.08, "initial_state_hash": "b21e43a5e73fb0cae9d3f09d0227fe2850be05e8dab551891667c22a806996e1", "final_state_hash": "476e05f0e0476f37f33adf7ed949652d450723920001c868c0a7c988be534a90", "case_id": "programming-train-00203" }, { "domain": "programming", "scenario": "首条订单去重", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 272.771, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "2af3602717b87c15bbc42b8282bd3fbdd64f6598f2772f4e57f025c32cdafe75", "case_id": "programming-train-00204" }, { "domain": "programming", "scenario": "首条订单去重", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 140.552, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "case_id": "programming-train-00205" }, { "domain": "programming", "scenario": "首条订单去重", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 142.235, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "case_id": "programming-train-00206" }, { "domain": "programming", "scenario": "首条订单去重", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 223.288, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "300eb55dee99cbc2c9ee206cff88357cff5d23e7e8049f3ff86ff1f625ccc565", "case_id": "programming-train-00207" }, { "domain": "programming", "scenario": "最新订单去重", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 279.232, "initial_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "final_state_hash": "fb3bdd48482185776addb39e9a38f80ee8bf748c206415b0080a64e89bff52d4", "case_id": "programming-train-00208" }, { "domain": "programming", "scenario": "最新订单去重", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 139.329, "initial_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "final_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "case_id": "programming-train-00209" }, { "domain": "programming", "scenario": "最新订单去重", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 141.934, "initial_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "final_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "case_id": "programming-train-00210" }, { "domain": "programming", "scenario": "最新订单去重", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 228.651, "initial_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "final_state_hash": "ba993ffe3a334167dab84709e6f038f5c9529f74da5871a00ded6e0e6fc92a8b", "case_id": "programming-train-00211" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 277.755, "initial_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "final_state_hash": "5441e9f871b4f34e81f7ba964e8f0853448c3439c0279d7fb1a2b6ad18ae7fed", "case_id": "programming-train-00212" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 141.705, "initial_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "final_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "case_id": "programming-train-00213" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 144.885, "initial_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "final_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "case_id": "programming-train-00214" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 225.527, "initial_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "final_state_hash": "95529e5e6abe05be2301c8c7fd38844acb1b72a6c4e7262df2b6a2cbdf42df67", "case_id": "programming-train-00215" }, { "domain": "programming", "scenario": "首条订单去重", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 275.129, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "2af3602717b87c15bbc42b8282bd3fbdd64f6598f2772f4e57f025c32cdafe75", "case_id": "programming-eval-00216" }, { "domain": "programming", "scenario": "首条订单去重", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 137.249, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "case_id": "programming-eval-00217" }, { "domain": "programming", "scenario": "首条订单去重", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 141.929, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "case_id": "programming-eval-00218" }, { "domain": "programming", "scenario": "首条订单去重", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 231.728, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "300eb55dee99cbc2c9ee206cff88357cff5d23e7e8049f3ff86ff1f625ccc565", "case_id": "programming-eval-00219" }, { "domain": "programming", "scenario": "最新订单去重", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 272.213, "initial_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "final_state_hash": "fb3bdd48482185776addb39e9a38f80ee8bf748c206415b0080a64e89bff52d4", "case_id": "programming-eval-00220" }, { "domain": "programming", "scenario": "最新订单去重", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 143.018, "initial_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "final_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "case_id": "programming-eval-00221" }, { "domain": "programming", "scenario": "最新订单去重", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 139.217, "initial_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "final_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "case_id": "programming-eval-00222" }, { "domain": "programming", "scenario": "最新订单去重", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 228.136, "initial_state_hash": "ca74a482e499c39e8850cdc361933c76f7001e230f061e29cd25e83266abad6f", "final_state_hash": "ba993ffe3a334167dab84709e6f038f5c9529f74da5871a00ded6e0e6fc92a8b", "case_id": "programming-eval-00223" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 274.677, "initial_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "final_state_hash": "5441e9f871b4f34e81f7ba964e8f0853448c3439c0279d7fb1a2b6ad18ae7fed", "case_id": "programming-eval-00224" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 139.203, "initial_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "final_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "case_id": "programming-eval-00225" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 142.151, "initial_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "final_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "case_id": "programming-eval-00226" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 225.854, "initial_state_hash": "1e91f8066622754e880b1a714e451200040f1f9b33f92aa3d349c1c691c2bdfd", "final_state_hash": "95529e5e6abe05be2301c8c7fd38844acb1b72a6c4e7262df2b6a2cbdf42df67", "case_id": "programming-eval-00227" }, { "domain": "programming", "scenario": "首条订单去重", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 273.78, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "2af3602717b87c15bbc42b8282bd3fbdd64f6598f2772f4e57f025c32cdafe75", "case_id": "programming-eval-00228" }, { "domain": "programming", "scenario": "首条订单去重", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 171.492, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "case_id": "programming-eval-00229" }, { "domain": "programming", "scenario": "首条订单去重", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 162.62, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "case_id": "programming-eval-00230" }, { "domain": "programming", "scenario": "首条订单去重", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 230.651, "initial_state_hash": "52d1dff6876eeda8866ad09314b9690102ad503f18a061fa85a95326e4ff97bf", "final_state_hash": "300eb55dee99cbc2c9ee206cff88357cff5d23e7e8049f3ff86ff1f625ccc565", "case_id": "programming-eval-00231" }, { "domain": "programming", "scenario": "最新订单去重", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 273.111, "initial_state_hash": "73bfafeb8d91ac0dff8589231d1dd2028ab5d4e404ec46a76df314f956538658", "final_state_hash": "6b4b746f03914d739e8130e3f12e031329425024600db1127977c2f823c95921", "case_id": "programming-eval-00232" }, { "domain": "programming", "scenario": "最新订单去重", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 144.227, "initial_state_hash": "73bfafeb8d91ac0dff8589231d1dd2028ab5d4e404ec46a76df314f956538658", "final_state_hash": "73bfafeb8d91ac0dff8589231d1dd2028ab5d4e404ec46a76df314f956538658", "case_id": "programming-eval-00233" }, { "domain": "programming", "scenario": "最新订单去重", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 147.291, "initial_state_hash": "73bfafeb8d91ac0dff8589231d1dd2028ab5d4e404ec46a76df314f956538658", "final_state_hash": "73bfafeb8d91ac0dff8589231d1dd2028ab5d4e404ec46a76df314f956538658", "case_id": "programming-eval-00234" }, { "domain": "programming", "scenario": "最新订单去重", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 224.844, "initial_state_hash": "73bfafeb8d91ac0dff8589231d1dd2028ab5d4e404ec46a76df314f956538658", "final_state_hash": "1d54a020bfa234cb8347ccc5231995a6a2511917c8817b58feefc5a5ed0b17d1", "case_id": "programming-eval-00235" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 272.811, "initial_state_hash": "ca918bccc8f9976429d9424085298789aa4865be14303ab1c140f324c7c66093", "final_state_hash": "03de05b61207a44863c220ca4f2f2db3ddefdae73a255e0b3b067f792c5ade25", "case_id": "programming-eval-00236" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 140.672, "initial_state_hash": "ca918bccc8f9976429d9424085298789aa4865be14303ab1c140f324c7c66093", "final_state_hash": "ca918bccc8f9976429d9424085298789aa4865be14303ab1c140f324c7c66093", "case_id": "programming-eval-00237" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 14, "terminated": false, "truncated": true, "invalid_actions": 14, "failed_checks": [ "hidden_tests_passed", "current_source_tested", "change_explained" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 150.858, "initial_state_hash": "ca918bccc8f9976429d9424085298789aa4865be14303ab1c140f324c7c66093", "final_state_hash": "ca918bccc8f9976429d9424085298789aa4865be14303ab1c140f324c7c66093", "case_id": "programming-eval-00238" }, { "domain": "programming", "scenario": "已结算订单汇总", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 4, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "hidden_tests_passed" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 230.634, "initial_state_hash": "ca918bccc8f9976429d9424085298789aa4865be14303ab1c140f324c7c66093", "final_state_hash": "e888056f4aa4e9bc9f0f1eaf237303d06360316849aaf4002f2575712b7c8780", "case_id": "programming-eval-00239" }, { "domain": "support", "scenario": "标准退货政策", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.975, "initial_state_hash": "d056a348d55a4ba8998aa05b6ec6fbc5e99389994b69d8a0db243a80b77b6a9a", "final_state_hash": "d1eb5cf4439a39a3445c8e414ebcf055fe9f34a56fab5b76fe5231d1a11783f1", "case_id": "support-train-00240" }, { "domain": "support", "scenario": "标准退货政策", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.221, "initial_state_hash": "d056a348d55a4ba8998aa05b6ec6fbc5e99389994b69d8a0db243a80b77b6a9a", "final_state_hash": "d056a348d55a4ba8998aa05b6ec6fbc5e99389994b69d8a0db243a80b77b6a9a", "case_id": "support-train-00241" }, { "domain": "support", "scenario": "标准退货政策", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.726, "initial_state_hash": "d056a348d55a4ba8998aa05b6ec6fbc5e99389994b69d8a0db243a80b77b6a9a", "final_state_hash": "d056a348d55a4ba8998aa05b6ec6fbc5e99389994b69d8a0db243a80b77b6a9a", "case_id": "support-train-00242" }, { "domain": "support", "scenario": "标准退货政策", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.421, "initial_state_hash": "d056a348d55a4ba8998aa05b6ec6fbc5e99389994b69d8a0db243a80b77b6a9a", "final_state_hash": "d056a348d55a4ba8998aa05b6ec6fbc5e99389994b69d8a0db243a80b77b6a9a", "case_id": "support-train-00243" }, { "domain": "support", "scenario": "损坏换货政策", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.888, "initial_state_hash": "2039eea9be50d663f61fddc3c58f7ea915b1b53cb53e3e6b5053a7ce5b4cc75d", "final_state_hash": "09f6de94395ce4174407f166e98b0a11dd199e976ea0bc1119a435726bbdfdb7", "case_id": "support-train-00244" }, { "domain": "support", "scenario": "损坏换货政策", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.209, "initial_state_hash": "2039eea9be50d663f61fddc3c58f7ea915b1b53cb53e3e6b5053a7ce5b4cc75d", "final_state_hash": "2039eea9be50d663f61fddc3c58f7ea915b1b53cb53e3e6b5053a7ce5b4cc75d", "case_id": "support-train-00245" }, { "domain": "support", "scenario": "损坏换货政策", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.685, "initial_state_hash": "2039eea9be50d663f61fddc3c58f7ea915b1b53cb53e3e6b5053a7ce5b4cc75d", "final_state_hash": "2039eea9be50d663f61fddc3c58f7ea915b1b53cb53e3e6b5053a7ce5b4cc75d", "case_id": "support-train-00246" }, { "domain": "support", "scenario": "损坏换货政策", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.419, "initial_state_hash": "2039eea9be50d663f61fddc3c58f7ea915b1b53cb53e3e6b5053a7ce5b4cc75d", "final_state_hash": "2039eea9be50d663f61fddc3c58f7ea915b1b53cb53e3e6b5053a7ce5b4cc75d", "case_id": "support-train-00247" }, { "domain": "support", "scenario": "延长退货政策", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.873, "initial_state_hash": "c5cd090833d8c3199c663d03b1ff90942b52946bb7197a9fe7f887ef705ca429", "final_state_hash": "774f0dad284e7c84bf33af07158ff963d300e3170114b93ffba1da41e2667c96", "case_id": "support-train-00248" }, { "domain": "support", "scenario": "延长退货政策", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.206, "initial_state_hash": "c5cd090833d8c3199c663d03b1ff90942b52946bb7197a9fe7f887ef705ca429", "final_state_hash": "c5cd090833d8c3199c663d03b1ff90942b52946bb7197a9fe7f887ef705ca429", "case_id": "support-train-00249" }, { "domain": "support", "scenario": "延长退货政策", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.661, "initial_state_hash": "c5cd090833d8c3199c663d03b1ff90942b52946bb7197a9fe7f887ef705ca429", "final_state_hash": "c5cd090833d8c3199c663d03b1ff90942b52946bb7197a9fe7f887ef705ca429", "case_id": "support-train-00250" }, { "domain": "support", "scenario": "延长退货政策", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.411, "initial_state_hash": "c5cd090833d8c3199c663d03b1ff90942b52946bb7197a9fe7f887ef705ca429", "final_state_hash": "c5cd090833d8c3199c663d03b1ff90942b52946bb7197a9fe7f887ef705ca429", "case_id": "support-train-00251" }, { "domain": "support", "scenario": "标准退货政策", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.873, "initial_state_hash": "c1d1ab2e0f7cc080477333141b6e5266736c18638cef4b78fda5f905e0245872", "final_state_hash": "e0040cd183b75d02e850ce123fd25537ab67ff22aa44dc2261fb93847e0d2833", "case_id": "support-train-00252" }, { "domain": "support", "scenario": "标准退货政策", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.206, "initial_state_hash": "c1d1ab2e0f7cc080477333141b6e5266736c18638cef4b78fda5f905e0245872", "final_state_hash": "c1d1ab2e0f7cc080477333141b6e5266736c18638cef4b78fda5f905e0245872", "case_id": "support-train-00253" }, { "domain": "support", "scenario": "标准退货政策", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.667, "initial_state_hash": "c1d1ab2e0f7cc080477333141b6e5266736c18638cef4b78fda5f905e0245872", "final_state_hash": "c1d1ab2e0f7cc080477333141b6e5266736c18638cef4b78fda5f905e0245872", "case_id": "support-train-00254" }, { "domain": "support", "scenario": "标准退货政策", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.423, "initial_state_hash": "c1d1ab2e0f7cc080477333141b6e5266736c18638cef4b78fda5f905e0245872", "final_state_hash": "c1d1ab2e0f7cc080477333141b6e5266736c18638cef4b78fda5f905e0245872", "case_id": "support-train-00255" }, { "domain": "support", "scenario": "损坏换货政策", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.87, "initial_state_hash": "d9d8d4ede50e3bcd8f5a05f95117db4615457ae5f80bf81f1b007311a9ebf71e", "final_state_hash": "72f1e4a2ac7c28701266a4e4b1a39e21cd9740d6ee8b4a2360abea2593184339", "case_id": "support-train-00256" }, { "domain": "support", "scenario": "损坏换货政策", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.207, "initial_state_hash": "d9d8d4ede50e3bcd8f5a05f95117db4615457ae5f80bf81f1b007311a9ebf71e", "final_state_hash": "d9d8d4ede50e3bcd8f5a05f95117db4615457ae5f80bf81f1b007311a9ebf71e", "case_id": "support-train-00257" }, { "domain": "support", "scenario": "损坏换货政策", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.659, "initial_state_hash": "d9d8d4ede50e3bcd8f5a05f95117db4615457ae5f80bf81f1b007311a9ebf71e", "final_state_hash": "d9d8d4ede50e3bcd8f5a05f95117db4615457ae5f80bf81f1b007311a9ebf71e", "case_id": "support-train-00258" }, { "domain": "support", "scenario": "损坏换货政策", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.416, "initial_state_hash": "d9d8d4ede50e3bcd8f5a05f95117db4615457ae5f80bf81f1b007311a9ebf71e", "final_state_hash": "d9d8d4ede50e3bcd8f5a05f95117db4615457ae5f80bf81f1b007311a9ebf71e", "case_id": "support-train-00259" }, { "domain": "support", "scenario": "延长退货政策", "split": "train", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.877, "initial_state_hash": "8b1d574d6718bfa6b34e9459476cc5b41d52768a96d3943aa10e9b6cc4eb565e", "final_state_hash": "65cd8ed89df8570ac26af350989e9d177aa12b1c84853133fc389763ce00f49c", "case_id": "support-train-00260" }, { "domain": "support", "scenario": "延长退货政策", "split": "train", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.213, "initial_state_hash": "8b1d574d6718bfa6b34e9459476cc5b41d52768a96d3943aa10e9b6cc4eb565e", "final_state_hash": "8b1d574d6718bfa6b34e9459476cc5b41d52768a96d3943aa10e9b6cc4eb565e", "case_id": "support-train-00261" }, { "domain": "support", "scenario": "延长退货政策", "split": "train", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.668, "initial_state_hash": "8b1d574d6718bfa6b34e9459476cc5b41d52768a96d3943aa10e9b6cc4eb565e", "final_state_hash": "8b1d574d6718bfa6b34e9459476cc5b41d52768a96d3943aa10e9b6cc4eb565e", "case_id": "support-train-00262" }, { "domain": "support", "scenario": "延长退货政策", "split": "train", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.408, "initial_state_hash": "8b1d574d6718bfa6b34e9459476cc5b41d52768a96d3943aa10e9b6cc4eb565e", "final_state_hash": "8b1d574d6718bfa6b34e9459476cc5b41d52768a96d3943aa10e9b6cc4eb565e", "case_id": "support-train-00263" }, { "domain": "support", "scenario": "标准退货政策", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.871, "initial_state_hash": "b67e267f49368cf1737f7383f88460d046361efc38c0b8ea4426a930d566e1af", "final_state_hash": "af6fc6010d1352790753de02f3358312248812e8ee6ebbb9838d099472035ef2", "case_id": "support-eval-00264" }, { "domain": "support", "scenario": "标准退货政策", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.209, "initial_state_hash": "b67e267f49368cf1737f7383f88460d046361efc38c0b8ea4426a930d566e1af", "final_state_hash": "b67e267f49368cf1737f7383f88460d046361efc38c0b8ea4426a930d566e1af", "case_id": "support-eval-00265" }, { "domain": "support", "scenario": "标准退货政策", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.692, "initial_state_hash": "b67e267f49368cf1737f7383f88460d046361efc38c0b8ea4426a930d566e1af", "final_state_hash": "b67e267f49368cf1737f7383f88460d046361efc38c0b8ea4426a930d566e1af", "case_id": "support-eval-00266" }, { "domain": "support", "scenario": "标准退货政策", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.41, "initial_state_hash": "b67e267f49368cf1737f7383f88460d046361efc38c0b8ea4426a930d566e1af", "final_state_hash": "b67e267f49368cf1737f7383f88460d046361efc38c0b8ea4426a930d566e1af", "case_id": "support-eval-00267" }, { "domain": "support", "scenario": "损坏换货政策", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.871, "initial_state_hash": "df992a7f39845369c8505621882c26b7854b5b6668c1b23d85e954eeaa934594", "final_state_hash": "4a44a9b0d0a4193d44745ed9b6f23b67724fdd33a9293d6f792c3353d071f03d", "case_id": "support-eval-00268" }, { "domain": "support", "scenario": "损坏换货政策", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.205, "initial_state_hash": "df992a7f39845369c8505621882c26b7854b5b6668c1b23d85e954eeaa934594", "final_state_hash": "df992a7f39845369c8505621882c26b7854b5b6668c1b23d85e954eeaa934594", "case_id": "support-eval-00269" }, { "domain": "support", "scenario": "损坏换货政策", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.68, "initial_state_hash": "df992a7f39845369c8505621882c26b7854b5b6668c1b23d85e954eeaa934594", "final_state_hash": "df992a7f39845369c8505621882c26b7854b5b6668c1b23d85e954eeaa934594", "case_id": "support-eval-00270" }, { "domain": "support", "scenario": "损坏换货政策", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.409, "initial_state_hash": "df992a7f39845369c8505621882c26b7854b5b6668c1b23d85e954eeaa934594", "final_state_hash": "df992a7f39845369c8505621882c26b7854b5b6668c1b23d85e954eeaa934594", "case_id": "support-eval-00271" }, { "domain": "support", "scenario": "延长退货政策", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.865, "initial_state_hash": "eb1da266cb442fdac61a7aeaa01195f4909e48c9e8470a35e82b7b79618e1b89", "final_state_hash": "3084a90a711c844ef473bc0073c2d34084ff808590e31e6d4097ec158dfb266d", "case_id": "support-eval-00272" }, { "domain": "support", "scenario": "延长退货政策", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.215, "initial_state_hash": "eb1da266cb442fdac61a7aeaa01195f4909e48c9e8470a35e82b7b79618e1b89", "final_state_hash": "eb1da266cb442fdac61a7aeaa01195f4909e48c9e8470a35e82b7b79618e1b89", "case_id": "support-eval-00273" }, { "domain": "support", "scenario": "延长退货政策", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.684, "initial_state_hash": "eb1da266cb442fdac61a7aeaa01195f4909e48c9e8470a35e82b7b79618e1b89", "final_state_hash": "eb1da266cb442fdac61a7aeaa01195f4909e48c9e8470a35e82b7b79618e1b89", "case_id": "support-eval-00274" }, { "domain": "support", "scenario": "延长退货政策", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.411, "initial_state_hash": "eb1da266cb442fdac61a7aeaa01195f4909e48c9e8470a35e82b7b79618e1b89", "final_state_hash": "eb1da266cb442fdac61a7aeaa01195f4909e48c9e8470a35e82b7b79618e1b89", "case_id": "support-eval-00275" }, { "domain": "support", "scenario": "标准退货政策", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.864, "initial_state_hash": "179aa180ac2d79314522309902c114ef1e3e001655c3994e843e2b788df8c9ca", "final_state_hash": "a415d40c5e53ec8d5128fa1935ab9baa0300c2c0489da5d661f9e75d670e3d49", "case_id": "support-eval-00276" }, { "domain": "support", "scenario": "标准退货政策", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.204, "initial_state_hash": "179aa180ac2d79314522309902c114ef1e3e001655c3994e843e2b788df8c9ca", "final_state_hash": "179aa180ac2d79314522309902c114ef1e3e001655c3994e843e2b788df8c9ca", "case_id": "support-eval-00277" }, { "domain": "support", "scenario": "标准退货政策", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.675, "initial_state_hash": "179aa180ac2d79314522309902c114ef1e3e001655c3994e843e2b788df8c9ca", "final_state_hash": "179aa180ac2d79314522309902c114ef1e3e001655c3994e843e2b788df8c9ca", "case_id": "support-eval-00278" }, { "domain": "support", "scenario": "标准退货政策", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.417, "initial_state_hash": "179aa180ac2d79314522309902c114ef1e3e001655c3994e843e2b788df8c9ca", "final_state_hash": "179aa180ac2d79314522309902c114ef1e3e001655c3994e843e2b788df8c9ca", "case_id": "support-eval-00279" }, { "domain": "support", "scenario": "损坏换货政策", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.863, "initial_state_hash": "71b7176ddb341527eae181ea07fc1059e9f75d1e99b4888bf57a4b8c6736b4c4", "final_state_hash": "360a35b7e83173009119fbd47113c796d864a8722515de910455631f95c9a11b", "case_id": "support-eval-00280" }, { "domain": "support", "scenario": "损坏换货政策", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.211, "initial_state_hash": "71b7176ddb341527eae181ea07fc1059e9f75d1e99b4888bf57a4b8c6736b4c4", "final_state_hash": "71b7176ddb341527eae181ea07fc1059e9f75d1e99b4888bf57a4b8c6736b4c4", "case_id": "support-eval-00281" }, { "domain": "support", "scenario": "损坏换货政策", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.661, "initial_state_hash": "71b7176ddb341527eae181ea07fc1059e9f75d1e99b4888bf57a4b8c6736b4c4", "final_state_hash": "71b7176ddb341527eae181ea07fc1059e9f75d1e99b4888bf57a4b8c6736b4c4", "case_id": "support-eval-00282" }, { "domain": "support", "scenario": "损坏换货政策", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.408, "initial_state_hash": "71b7176ddb341527eae181ea07fc1059e9f75d1e99b4888bf57a4b8c6736b4c4", "final_state_hash": "71b7176ddb341527eae181ea07fc1059e9f75d1e99b4888bf57a4b8c6736b4c4", "case_id": "support-eval-00283" }, { "domain": "support", "scenario": "延长退货政策", "split": "eval", "policy": "reference", "expected_reward": 1, "reward": 1, "steps": 5, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.883, "initial_state_hash": "a8328617ad867c44a333117f4a17355394e24c011014f950d2af1a5494930e7c", "final_state_hash": "db69a7f0e771f59f6be72b9e105375d32c488d2170494f0043ac55c89edb9f81", "case_id": "support-eval-00284" }, { "domain": "support", "scenario": "延长退货政策", "split": "eval", "policy": "empty", "expected_reward": 0, "reward": 0, "steps": 1, "terminated": true, "truncated": false, "invalid_actions": 0, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.204, "initial_state_hash": "a8328617ad867c44a333117f4a17355394e24c011014f950d2af1a5494930e7c", "final_state_hash": "a8328617ad867c44a333117f4a17355394e24c011014f950d2af1a5494930e7c", "case_id": "support-eval-00285" }, { "domain": "support", "scenario": "延长退货政策", "split": "eval", "policy": "invalid", "expected_reward": 0, "reward": 0, "steps": 16, "terminated": false, "truncated": true, "invalid_actions": 16, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 1.673, "initial_state_hash": "a8328617ad867c44a333117f4a17355394e24c011014f950d2af1a5494930e7c", "final_state_hash": "a8328617ad867c44a333117f4a17355394e24c011014f950d2af1a5494930e7c", "case_id": "support-eval-00286" }, { "domain": "support", "scenario": "延长退货政策", "split": "eval", "policy": "shortcut", "expected_reward": 0, "reward": 0, "steps": 3, "terminated": true, "truncated": false, "invalid_actions": 2, "failed_checks": [ "order_and_policy_checked", "policy_compliant_resolution", "customer_reply_present" ], "execution_error": null, "replay_consistent": true, "qa_passed": true, "elapsed_ms": 0.406, "initial_state_hash": "a8328617ad867c44a333117f4a17355394e24c011014f950d2af1a5494930e7c", "final_state_hash": "a8328617ad867c44a333117f4a17355394e24c011014f950d2af1a5494930e7c", "case_id": "support-eval-00287" } ], "limitations": [ "固定合成策略，不是模型评测或真实专家数据。", "零误放只覆盖本次明确列出的攻击，不证明不存在其他奖励漏洞。", "训练和评估分区使用不同种子命名空间，但仍共享任务模板，不代表分布外泛化。" ] }