{ "schema_version": "shopsimrl-behavior-analysis-v1", "extraction": { "schema_version": "shopsimrl-behavior-extraction-v1", "train_files": 20480, "train_turns": 110997, "train_manifest_sha256": "e61784f7d981008acbb4e3a4e00b30fc3bf44a44fd2423836529752b795572dd", "tokenizer": { "source": "https://huggingface.co/Qwen/Qwen3.5-4B/resolve/main/tokenizer.json", "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42", "tokenizers_version": "0.20.1", "add_special_tokens": false, "basis": "re-encoding saved raw_output, NOT original sampled token IDs", "matches_training_tokenizer_hash": "unverified; historical tokenizer hash not recorded" }, "eval_sources": { "val_base": { "source": "runs/qwen35-4b-val-base-free/traces.jsonl", "sha256": "4ce37ba2e4babf418a0d9958786b6348f6af9d93fc01514970b55d8ed9e6e8e4" }, "val_20": { "source": "runs/round-000-bare-val/traces.jsonl", "sha256": "1d332ffde24c40ab1b8b0a7162c61a6ff9fc47a9cc9217586def2aed67ecc7ee" }, "val_40": { "source": "runs/round-001-bare-val/traces.jsonl", "sha256": "80fb5c17753a89861683c6f696e345dcb1cb79217fc5729bfc32c296cc40ffd3" }, "val_60": { "source": "runs/round-002-bare-val/traces.jsonl", "sha256": "5760c541078a97225b9f3a70cb3b6e903e851f37f6e8745e788ceeef5d3d5fa0" }, "val_80": { "source": "runs/round-003-bare-val/traces.jsonl", "sha256": "f3a187152a0a766436be8600f33418606a87f0743ddec1f190a285df34bc8df3" }, "test_base_free": { "source": "runs/qwen35-4b-test-base-free/traces.jsonl", "sha256": "c65c65d742095511d988464a51622f2d77590de33652c63edfca96e1f05bef54" }, "test_base_s0": { "source": "runs/qwen35-4b-test-base-s0/traces.jsonl", "sha256": "bdd518deda2e2ed17793c70c4b34bdc2581647fe65e7f47c4445be98b0d85fd9" }, "test_iter80_free": { "source": "runs/qwen35-4b-test-iter80-free/traces.jsonl", "sha256": "b48048e3bb50c929bab984ce892015ab25fdb3d30d0ec1886efd9e294e0ec41c" }, "test_iter80_st": { "source": "runs/qwen35-4b-test-iter80-st/traces.jsonl", "sha256": "ca7aa50cce79cac046e06eb868c80ab15fb05dd0735748cc84a3fbdcdfc59ea2" } } }, "rounds": [ { "round": 0, "all": { "n": 5061, "tasks": 636, "metrics": { "model_steps": { "n": 5061, "mean": 5.343805572021339, "median": 4.0, "p90": 8.0 }, "actions": { "n": 5061, "mean": 5.190871369294606, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 5061, "mean": 4.981426595534479, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 5061, "mean": 277.2977646753892, "median": 257.0, "p90": 394.25 }, "generated_tokens": { "n": 5061, "mean": 1447.7131001778305, "median": 1154.0, "p90": 2348.0 }, "first_turn_tokens": { "n": 5061, "mean": 205.1311993677139, "median": 195.0, "p90": 293.0 }, "searches": { "n": 5061, "mean": 1.1886978857933215, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 5061, "mean": 1.1803991306065995, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 5061, "mean": 1.1531317921359414, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 5061, "mean": 1.1232957913455839, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 5061, "mean": 0.02983600079035764, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 5061, "mean": 0.18948824343015214, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 5061, "mean": 0.11736810906935388, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 5061, "mean": 1.3683066587630903, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 5061, "mean": 0.07271290258842127, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 5061, "mean": 0.11934400316143055, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 5061, "mean": 0.20944477376012646, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 5061, "mean": 352.8574973684667, "median": 330.0, "p90": 511.0 }, "first_product_rank": { "n": 5001, "mean": 4.26374725054989, "median": 1.0, "p90": 11.0 } }, "success_rate": 0.6390041493775933, "reward_mean": 0.6519776183661778, "pooled_tokens_per_turn": 270.914254021076, "token_complete_episodes": 5061, "rates": { "protocol_errors": 0.05710333926101561, "product_revisits": 0.02410590792333531, "option_changes": 0.02983600079035764, "backtracks": 0.11321873147599289 }, "terminations": { "purchase": 4881, "generation_length": 170, "action_limit": 10 } }, "success": { "n": 3234, "tasks": 532, "metrics": { "model_steps": { "n": 3234, "mean": 4.543290043290043, "median": 4.0, "p90": 5.0 }, "actions": { "n": 3234, "mean": 4.512368583797155, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 3234, "mean": 4.499072356215214, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 3234, "mean": 259.01374104357757, "median": 243.0, "p90": 358.92500000000007 }, "generated_tokens": { "n": 3234, "mean": 1167.9774273345702, "median": 1042.5, "p90": 1717.0 }, "first_turn_tokens": { "n": 3234, "mean": 201.5834879406308, "median": 191.0, "p90": 287.0 }, "searches": { "n": 3234, "mean": 1.0655534941249227, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 3234, "mean": 1.064625850340136, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 3234, "mean": 1.0606060606060606, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 3234, "mean": 1.0522572665429808, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 3234, "mean": 0.008348794063079777, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 3234, "mean": 0.0655534941249227, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 3234, "mean": 0.031849103277674706, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 3234, "mean": 1.2755102040816326, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 3234, "mean": 0.027829313543599257, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 3234, "mean": 0.030921459492888066, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 3234, "mean": 0.013296227581941867, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 3234, "mean": 331.17217840175994, "median": 311.325, "p90": 464.67500000000007 }, "first_product_rank": { "n": 3234, "mean": 3.3194186765615337, "median": 1.0, "p90": 8.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 257.0774518478187, "token_complete_episodes": 3234, "rates": { "protocol_errors": 0.025664811379097093, "product_revisits": 0.007730364873222016, "option_changes": 0.017006802721088437, "backtracks": 0.04823747680890538 }, "terminations": { "purchase": 3234 } }, "failure": { "n": 1827, "tasks": 387, "metrics": { "model_steps": { "n": 1827, "mean": 6.760810071154899, "median": 4.0, "p90": 13.0 }, "actions": { "n": 1827, "mean": 6.391899288451013, "median": 4.0, "p90": 12.0 }, "valid_actions": { "n": 1827, "mean": 5.835249042145594, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 1827, "mean": 309.6625881156075, "median": 284.8181818181818, "p90": 435.0500000000003 }, "generated_tokens": { "n": 1827, "mean": 1942.8773946360154, "median": 1439.0, "p90": 3784.000000000001 }, "first_turn_tokens": { "n": 1827, "mean": 211.4110563765736, "median": 202.0, "p90": 303.4000000000001 }, "searches": { "n": 1827, "mean": 1.4066776135741652, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 1827, "mean": 1.3853311439518337, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 1827, "mean": 1.3169129720853858, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 1827, "mean": 1.2490421455938698, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 1827, "mean": 0.06787082649151614, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 1827, "mean": 0.4088669950738916, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 1827, "mean": 0.2687465790914067, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 1827, "mean": 1.5325670498084292, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 1827, "mean": 0.15216201423097975, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 1827, "mean": 0.27586206896551724, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 1827, "mean": 0.5566502463054187, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 1827, "mean": 391.24300450493627, "median": 369.0, "p90": 567.3000000000001 }, "first_product_rank": { "n": 1767, "mean": 5.992076966610074, "median": 2.0, "p90": 16.0 } }, "success_rate": 0.0, "reward_mean": 0.0359380003017113, "pooled_tokens_per_turn": 287.3734617875648, "token_complete_episodes": 1827, "rates": { "protocol_errors": 0.11275314723590586, "product_revisits": 0.05309250136836344, "option_changes": 0.052545155993431854, "backtracks": 0.22824302134646962 }, "terminations": { "purchase": 1647, "generation_length": 170, "action_limit": 10 } }, "technical_n": 59, "skill_free": { "n": 985, "tasks": 124, "metrics": { "model_steps": { "n": 985, "mean": 5.233502538071066, "median": 4.0, "p90": 8.0 }, "actions": { "n": 985, "mean": 5.094416243654822, "median": 4.0, "p90": 7.0 }, "valid_actions": { "n": 985, "mean": 4.8446700507614215, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 985, "mean": 259.72935862585433, "median": 235.25, "p90": 362.78000000000003 }, "generated_tokens": { "n": 985, "mean": 1336.5796954314721, "median": 1049.0, "p90": 2019.0000000000002 }, "first_turn_tokens": { "n": 985, "mean": 172.8497461928934, "median": 163.0, "p90": 233.60000000000002 }, "searches": { "n": 985, "mean": 1.1421319796954315, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 985, "mean": 1.1370558375634519, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 985, "mean": 1.1096446700507614, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 985, "mean": 1.0903553299492386, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 985, "mean": 0.019289340101522844, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 985, "mean": 0.14213197969543148, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 985, "mean": 0.116751269035533, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 985, "mean": 1.366497461928934, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 985, "mean": 0.05685279187817259, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 985, "mean": 0.10862944162436548, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 985, "mean": 0.24974619289340103, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 985, "mean": 321.5412742992663, "median": 293.6, "p90": 472.15000000000003 }, "first_product_rank": { "n": 968, "mean": 4.071280991735537, "median": 1.0, "p90": 11.0 } }, "success_rate": 0.6538071065989848, "reward_mean": 0.6660059221658206, "pooled_tokens_per_turn": 255.38913676042677, "token_complete_episodes": 985, "rates": { "protocol_errors": 0.03857868020304569, "product_revisits": 0.017258883248730966, "option_changes": 0.03350253807106599, "backtracks": 0.09137055837563451 }, "terminations": { "purchase": 953, "generation_length": 30, "action_limit": 2 } }, "assisted": { "n": 4076, "tasks": 512, "metrics": { "model_steps": { "n": 4076, "mean": 5.370461236506379, "median": 4.0, "p90": 8.0 }, "actions": { "n": 4076, "mean": 5.214180569185476, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 4076, "mean": 5.014474975466143, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 4076, "mean": 281.5433191304412, "median": 263.0, "p90": 400.0 }, "generated_tokens": { "n": 4076, "mean": 1474.5694308145241, "median": 1180.5, "p90": 2433.5 }, "first_turn_tokens": { "n": 4076, "mean": 212.93228655544652, "median": 204.0, "p90": 305.0 }, "searches": { "n": 4076, "mean": 1.1999509322865554, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 4076, "mean": 1.190873405299313, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 4076, "mean": 1.163640824337586, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 4076, "mean": 1.1312561334641806, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 4076, "mean": 0.0323846908734053, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 4076, "mean": 0.2009322865554465, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 4076, "mean": 0.11751717369970559, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 4076, "mean": 1.3687438665358194, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 4076, "mean": 0.07654563297350343, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 4076, "mean": 0.12193326790971541, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 4076, "mean": 0.1997055937193327, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 4076, "mean": 360.4253285076135, "median": 339.0625, "p90": 522.875 }, "first_product_rank": { "n": 4033, "mean": 4.309942970493429, "median": 1.0, "p90": 11.0 } }, "success_rate": 0.6354268891069677, "reward_mean": 0.6485875596707295, "pooled_tokens_per_turn": 274.57035175879395, "token_complete_episodes": 4076, "rates": { "protocol_errors": 0.06157998037291462, "product_revisits": 0.025760549558390577, "option_changes": 0.028949950932286556, "backtracks": 0.11849852796859667 }, "terminations": { "purchase": 3928, "generation_length": 140, "action_limit": 8 } } }, { "round": 1, "all": { "n": 5120, "tasks": 640, "metrics": { "model_steps": { "n": 5120, "mean": 5.5150390625, "median": 4.0, "p90": 8.0 }, "actions": { "n": 5120, "mean": 5.348828125, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 5120, "mean": 5.176953125, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 5120, "mean": 255.0209758485972, "median": 245.25, "p90": 337.0 }, "generated_tokens": { "n": 5120, "mean": 1410.2341796875, "median": 1071.0, "p90": 2441.2000000000007 }, "first_turn_tokens": { "n": 5120, "mean": 192.4193359375, "median": 187.0, "p90": 256.0 }, "searches": { "n": 5120, "mean": 1.2453125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 5120, "mean": 1.2419921875, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 5120, "mean": 1.2095703125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 5120, "mean": 1.1134765625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 5120, "mean": 0.09609375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 5120, "mean": 0.246875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 5120, "mean": 0.0435546875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 5120, "mean": 1.4734375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 5120, "mean": 0.121875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 5120, "mean": 0.1283203125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 5120, "mean": 0.171875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 5120, "mean": 330.4164173008669, "median": 316.75, "p90": 443.5 }, "first_product_rank": { "n": 5094, "mean": 1.967020023557126, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.8185546875, "reward_mean": 0.8251486967893218, "pooled_tokens_per_turn": 255.70701561780643, "token_complete_episodes": 5120, "rates": { "protocol_errors": 0.040234375, "product_revisits": 0.0591796875, "option_changes": 0.042578125, "backtracks": 0.1193359375 }, "terminations": { "purchase": 4906, "action_limit": 20, "generation_length": 194 } }, "success": { "n": 4191, "tasks": 583, "metrics": { "model_steps": { "n": 4191, "mean": 4.581961345740873, "median": 4.0, "p90": 5.0 }, "actions": { "n": 4191, "mean": 4.561202576950609, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 4191, "mean": 4.551658315437843, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 4191, "mean": 250.32302514994862, "median": 241.0, "p90": 329.75 }, "generated_tokens": { "n": 4191, "mean": 1153.4667143879742, "median": 1027.0, "p90": 1537.0 }, "first_turn_tokens": { "n": 4191, "mean": 192.44094488188978, "median": 188.0, "p90": 256.0 }, "searches": { "n": 4191, "mean": 1.0713433548079216, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 4191, "mean": 1.0696731090431877, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 4191, "mean": 1.066571224051539, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 4191, "mean": 1.0467668814125508, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 4191, "mean": 0.01980434263898831, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 4191, "mean": 0.07134335480792174, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 4191, "mean": 0.014077785731329038, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 4191, "mean": 1.3283225960391314, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 4191, "mean": 0.05440229062276306, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 4191, "mean": 0.020758768790264854, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 4191, "mean": 0.00954426151276545, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 4191, "mean": 323.62638181051904, "median": 311.2, "p90": 435.0 }, "first_product_rank": { "n": 4191, "mean": 1.697208303507516, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 251.74082174660208, "token_complete_episodes": 4191, "rates": { "protocol_errors": 0.017418277260796946, "product_revisits": 0.018611309949892626, "option_changes": 0.027678358387019805, "backtracks": 0.05440229062276306 }, "terminations": { "purchase": 4191 } }, "failure": { "n": 929, "tasks": 226, "metrics": { "model_steps": { "n": 929, "mean": 9.72443487621098, "median": 5.0, "p90": 22.0 }, "actions": { "n": 929, "mean": 8.902045209903122, "median": 5.0, "p90": 20.0 }, "valid_actions": { "n": 929, "mean": 7.997847147470399, "median": 5.0, "p90": 16.0 }, "tokens_per_turn": { "n": 929, "mean": 276.21485246650485, "median": 270.25, "p90": 361.275 }, "generated_tokens": { "n": 929, "mean": 2568.589881593111, "median": 1555.0, "p90": 5351.400000000001 }, "first_turn_tokens": { "n": 929, "mean": 192.32185145317547, "median": 185.0, "p90": 257.0 }, "searches": { "n": 929, "mean": 2.0301399354144243, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 929, "mean": 2.0193756727664156, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 929, "mean": 1.8546824542518838, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 929, "mean": 1.4144241119483316, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 929, "mean": 0.4402583423035522, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 929, "mean": 1.038751345532831, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 929, "mean": 0.17653390742734124, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 929, "mean": 2.1280947255113025, "median": 1.0, "p90": 4.0 }, "option_changes": { "n": 929, "mean": 0.426264800861141, "median": 0.0, "p90": 1.0 }, "protocol_errors": { "n": 929, "mean": 0.6135629709364908, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 929, "mean": 0.9041980624327234, "median": 0.0, "p90": 4.0 }, "reasoning_chars_per_turn": { "n": 929, "mean": 361.04832121910994, "median": 350.6, "p90": 482.8 }, "first_product_rank": { "n": 903, "mean": 3.219269102990033, "median": 1.0, "p90": 10.0 } }, "success_rate": 0.0, "reward_mean": 0.036341579721558194, "pooled_tokens_per_turn": 264.1377020146115, "token_complete_episodes": 929, "rates": { "protocol_errors": 0.14316469321851452, "product_revisits": 0.24219590958019377, "option_changes": 0.10979547900968784, "backtracks": 0.41227125941872983 }, "terminations": { "purchase": 715, "action_limit": 20, "generation_length": 194 } }, "technical_n": 0, "skill_free": { "n": 912, "tasks": 114, "metrics": { "model_steps": { "n": 912, "mean": 5.532894736842105, "median": 4.0, "p90": 8.0 }, "actions": { "n": 912, "mean": 5.3881578947368425, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 912, "mean": 5.206140350877193, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 912, "mean": 232.80300768585224, "median": 219.625, "p90": 311.4973684210526 }, "generated_tokens": { "n": 912, "mean": 1306.8377192982457, "median": 952.0, "p90": 2273.599999999999 }, "first_turn_tokens": { "n": 912, "mean": 158.72587719298247, "median": 151.5, "p90": 207.0 }, "searches": { "n": 912, "mean": 1.2598684210526316, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 912, "mean": 1.2576754385964912, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 912, "mean": 1.219298245614035, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 912, "mean": 1.1107456140350878, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 912, "mean": 0.10855263157894737, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 912, "mean": 0.26206140350877194, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 912, "mean": 0.03837719298245614, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 912, "mean": 1.468201754385965, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 912, "mean": 0.12719298245614036, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 912, "mean": 0.10964912280701754, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 912, "mean": 0.18201754385964913, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 912, "mean": 295.510936435984, "median": 277.5, "p90": 408.57499999999993 }, "first_product_rank": { "n": 908, "mean": 1.8832599118942732, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.819078947368421, "reward_mean": 0.8231237816764132, "pooled_tokens_per_turn": 236.19421323820848, "token_complete_episodes": 912, "rates": { "protocol_errors": 0.03508771929824561, "product_revisits": 0.0581140350877193, "option_changes": 0.0537280701754386, "backtracks": 0.12938596491228072 }, "terminations": { "purchase": 874, "generation_length": 32, "action_limit": 6 } }, "assisted": { "n": 4208, "tasks": 526, "metrics": { "model_steps": { "n": 4208, "mean": 5.511169201520913, "median": 4.0, "p90": 8.0 }, "actions": { "n": 4208, "mean": 5.340304182509506, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 4208, "mean": 5.1706273764258555, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 4208, "mean": 259.8362769332985, "median": 251.25, "p90": 339.25 }, "generated_tokens": { "n": 4208, "mean": 1432.6432984790874, "median": 1091.0, "p90": 2475.0 }, "first_turn_tokens": { "n": 4208, "mean": 199.72172053231938, "median": 195.5, "p90": 261.0 }, "searches": { "n": 4208, "mean": 1.242157794676806, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 4208, "mean": 1.2385931558935361, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 4208, "mean": 1.2074619771863118, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 4208, "mean": 1.1140684410646389, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 4208, "mean": 0.093393536121673, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 4208, "mean": 0.24358365019011408, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 4208, "mean": 0.04467680608365019, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 4208, "mean": 1.4745722433460076, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 4208, "mean": 0.12072243346007605, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 4208, "mean": 0.13236692015209126, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 4208, "mean": 0.16967680608365018, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 4208, "mean": 337.98148349591753, "median": 325.75, "p90": 449.75 }, "first_product_rank": { "n": 4186, "mean": 1.985188724319159, "median": 1.0, "p90": 4.0 } }, "success_rate": 0.8184410646387833, "reward_mean": 0.8255875567187354, "pooled_tokens_per_turn": 259.95269716700443, "token_complete_episodes": 4208, "rates": { "protocol_errors": 0.04134980988593156, "product_revisits": 0.0594106463878327, "option_changes": 0.04016159695817491, "backtracks": 0.11715779467680608 }, "terminations": { "purchase": 4032, "action_limit": 14, "generation_length": 162 } } }, { "round": 2, "all": { "n": 5120, "tasks": 640, "metrics": { "model_steps": { "n": 5120, "mean": 5.7255859375, "median": 4.0, "p90": 10.0 }, "actions": { "n": 5120, "mean": 5.6255859375, "median": 4.0, "p90": 10.0 }, "valid_actions": { "n": 5120, "mean": 5.50390625, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 5120, "mean": 190.09442939816049, "median": 182.2111111111111, "p90": 252.0 }, "generated_tokens": { "n": 5120, "mean": 1112.925390625, "median": 797.0, "p90": 2025.4000000000015 }, "first_turn_tokens": { "n": 5120, "mean": 134.946484375, "median": 128.0, "p90": 187.0 }, "searches": { "n": 5120, "mean": 1.3146484375, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 5120, "mean": 1.3015625, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 5120, "mean": 1.269921875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 5120, "mean": 1.1095703125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 5120, "mean": 0.1603515625, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 5120, "mean": 0.31640625, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 5120, "mean": 0.0453125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 5120, "mean": 1.601171875, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 5120, "mean": 0.14375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 5120, "mean": 0.0568359375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 5120, "mean": 0.1216796875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 5120, "mean": 236.7659592822378, "median": 226.6, "p90": 330.13750000000005 }, "first_product_rank": { "n": 5111, "mean": 1.4500097828213656, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.8361328125, "reward_mean": 0.846148538961039, "pooled_tokens_per_turn": 194.3775541531639, "token_complete_episodes": 5120, "rates": { "protocol_errors": 0.0466796875, "product_revisits": 0.10859375, "option_changes": 0.0533203125, "backtracks": 0.158203125 }, "terminations": { "purchase": 4897, "action_limit": 2, "generation_length": 221 } }, "success": { "n": 4281, "tasks": 586, "metrics": { "model_steps": { "n": 4281, "mean": 4.832282177061434, "median": 4.0, "p90": 7.0 }, "actions": { "n": 4281, "mean": 4.804952113992058, "median": 4.0, "p90": 7.0 }, "valid_actions": { "n": 4281, "mean": 4.788600794206961, "median": 4.0, "p90": 7.0 }, "tokens_per_turn": { "n": 4281, "mean": 184.65999799828143, "median": 178.75, "p90": 243.8 }, "generated_tokens": { "n": 4281, "mean": 903.3721093202523, "median": 767.0, "p90": 1349.0 }, "first_turn_tokens": { "n": 4281, "mean": 135.02919878533052, "median": 128.0, "p90": 188.0 }, "searches": { "n": 4281, "mean": 1.125437981779958, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 4281, "mean": 1.1170287316047653, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 4281, "mean": 1.1200654052791403, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 4281, "mean": 1.044382153702406, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 4281, "mean": 0.0756832515767344, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 4281, "mean": 0.12567157206260218, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 4281, "mean": 0.010044382153702406, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 4281, "mean": 1.407381452931558, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 4281, "mean": 0.07101144592384957, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 4281, "mean": 0.027330063069376315, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 4281, "mean": 0.01635131978509694, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 4281, "mean": 229.84780738121782, "median": 221.25, "p90": 317.6 }, "first_product_rank": { "n": 4281, "mean": 1.3277271665498716, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 186.9452313046841, "token_complete_episodes": 4281, "rates": { "protocol_errors": 0.02429338939500117, "product_revisits": 0.06470450829245504, "option_changes": 0.03620649380985751, "backtracks": 0.09320252277505256 }, "terminations": { "purchase": 4281 } }, "failure": { "n": 839, "tasks": 188, "metrics": { "model_steps": { "n": 839, "mean": 10.283671036948748, "median": 7.0, "p90": 22.0 }, "actions": { "n": 839, "mean": 9.812872467222885, "median": 7.0, "p90": 20.0 }, "valid_actions": { "n": 839, "mean": 9.153754469606675, "median": 7.0, "p90": 18.0 }, "tokens_per_turn": { "n": 839, "mean": 217.8236318092239, "median": 202.25, "p90": 279.20000000000005 }, "generated_tokens": { "n": 839, "mean": 2182.1716328963053, "median": 1578.0, "p90": 4658.6 }, "first_turn_tokens": { "n": 839, "mean": 134.5244338498212, "median": 127.0, "p90": 186.0 }, "searches": { "n": 839, "mean": 2.2800953516090585, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 839, "mean": 2.243146603098927, "median": 1.0, "p90": 5.0 }, "product_opens": { "n": 839, "mean": 2.034564958283671, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 839, "mean": 1.4421930870083433, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 839, "mean": 0.5923718712753278, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 839, "mean": 1.2896305125148988, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 839, "mean": 0.22526817640047675, "median": 0.0, "p90": 0.20000000000004547 }, "option_selections": { "n": 839, "mean": 2.5899880810488676, "median": 2.0, "p90": 5.0 }, "option_changes": { "n": 839, "mean": 0.5148986889153755, "median": 0.0, "p90": 1.2000000000000455 }, "protocol_errors": { "n": 839, "mean": 0.20738974970202623, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 839, "mean": 0.6591179976162098, "median": 0.0, "p90": 2.0 }, "reasoning_chars_per_turn": { "n": 839, "mean": 272.0658499714706, "median": 259.7142857142857, "p90": 371.4666666666667 }, "first_product_rank": { "n": 830, "mean": 2.080722891566265, "median": 1.0, "p90": 4.0 } }, "success_rate": 0.0, "reward_mean": 0.06112100057272881, "pooled_tokens_per_turn": 212.19772832637923, "token_complete_episodes": 839, "rates": { "protocol_errors": 0.16090584028605484, "product_revisits": 0.33253873659118, "option_changes": 0.14064362336114422, "backtracks": 0.4898688915375447 }, "terminations": { "purchase": 616, "action_limit": 2, "generation_length": 221 } }, "technical_n": 0, "skill_free": { "n": 1096, "tasks": 137, "metrics": { "model_steps": { "n": 1096, "mean": 5.563868613138686, "median": 4.0, "p90": 9.0 }, "actions": { "n": 1096, "mean": 5.476277372262774, "median": 4.0, "p90": 9.0 }, "valid_actions": { "n": 1096, "mean": 5.445255474452555, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 1096, "mean": 178.71361798655033, "median": 171.5, "p90": 228.0 }, "generated_tokens": { "n": 1096, "mean": 1019.0793795620438, "median": 742.5, "p90": 1851.5 }, "first_turn_tokens": { "n": 1096, "mean": 120.79653284671532, "median": 119.0, "p90": 152.0 }, "searches": { "n": 1096, "mean": 1.3166058394160585, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 1096, "mean": 1.3047445255474452, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 1096, "mean": 1.2627737226277371, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 1096, "mean": 1.1322992700729928, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 1096, "mean": 0.13047445255474452, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 1096, "mean": 0.3166058394160584, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 1096, "mean": 0.07208029197080291, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 1096, "mean": 1.510948905109489, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 1096, "mean": 0.06934306569343066, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 1096, "mean": 0.053832116788321165, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 1096, "mean": 0.03102189781021898, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 1096, "mean": 220.74696503554372, "median": 212.02884615384616, "p90": 296.79166666666663 }, "first_product_rank": { "n": 1091, "mean": 1.5765352887259394, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.8348540145985401, "reward_mean": 0.8467913625304136, "pooled_tokens_per_turn": 183.16021646441456, "token_complete_episodes": 1096, "rates": { "protocol_errors": 0.04927007299270073, "product_revisits": 0.09671532846715329, "option_changes": 0.037408759124087594, "backtracks": 0.15967153284671534 }, "terminations": { "purchase": 1059, "generation_length": 37 } }, "assisted": { "n": 4024, "tasks": 503, "metrics": { "model_steps": { "n": 4024, "mean": 5.769632206759443, "median": 4.0, "p90": 10.0 }, "actions": { "n": 4024, "mean": 5.666252485089463, "median": 4.0, "p90": 10.0 }, "valid_actions": { "n": 4024, "mean": 5.519880715705765, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 4024, "mean": 193.19417326176006, "median": 185.5, "p90": 255.7977272727273 }, "generated_tokens": { "n": 4024, "mean": 1138.4858349900596, "median": 811.0, "p90": 2089.2000000000016 }, "first_turn_tokens": { "n": 4024, "mean": 138.80044731610337, "median": 131.0, "p90": 197.0 }, "searches": { "n": 4024, "mean": 1.3141153081510935, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 4024, "mean": 1.3006958250497018, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 4024, "mean": 1.271868787276342, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 4024, "mean": 1.10337972166998, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 4024, "mean": 0.16848906560636182, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 4024, "mean": 0.316351888667992, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 4024, "mean": 0.03802186878727634, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 4024, "mean": 1.6257455268389662, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 4024, "mean": 0.16401590457256462, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 4024, "mean": 0.05765407554671968, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 4024, "mean": 0.1463717693836978, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 4024, "mean": 241.12898554823596, "median": 231.25, "p90": 336.34423076923076 }, "first_product_rank": { "n": 4020, "mean": 1.4156716417910449, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.8364811133200796, "reward_mean": 0.8459734558019846, "pooled_tokens_per_turn": 197.3238144463109, "token_complete_episodes": 4024, "rates": { "protocol_errors": 0.0459741550695825, "product_revisits": 0.11182902584493042, "option_changes": 0.05765407554671968, "backtracks": 0.15780318091451292 }, "terminations": { "purchase": 3838, "action_limit": 2, "generation_length": 184 } } }, { "round": 3, "all": { "n": 5120, "tasks": 640, "metrics": { "model_steps": { "n": 5120, "mean": 5.1177734375, "median": 4.0, "p90": 7.0 }, "actions": { "n": 5120, "mean": 4.9705078125, "median": 4.0, "p90": 7.0 }, "valid_actions": { "n": 5120, "mean": 4.8921875, "median": 4.0, "p90": 7.0 }, "tokens_per_turn": { "n": 5120, "mean": 221.2624335778477, "median": 205.18823529411765, "p90": 293.75 }, "generated_tokens": { "n": 5120, "mean": 1170.088671875, "median": 876.5, "p90": 1854.2000000000007 }, "first_turn_tokens": { "n": 5120, "mean": 173.377734375, "median": 167.0, "p90": 233.0 }, "searches": { "n": 5120, "mean": 1.1984375, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 5120, "mean": 1.1931640625, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 5120, "mean": 1.1564453125, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 5120, "mean": 1.0703125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 5120, "mean": 0.0861328125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 5120, "mean": 0.20078125, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 5120, "mean": 0.0392578125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 5120, "mean": 1.3298828125, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 5120, "mean": 0.056640625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 5120, "mean": 0.1146484375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 5120, "mean": 0.0783203125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 5120, "mean": 286.3611612565824, "median": 266.0, "p90": 402.0200000000001 }, "first_product_rank": { "n": 5103, "mean": 1.2218302959043699, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.8732421875, "reward_mean": 0.8803034319196428, "pooled_tokens_per_turn": 228.63237033927413, "token_complete_episodes": 5120, "rates": { "protocol_errors": 0.0892578125, "product_revisits": 0.058203125, "option_changes": 0.0328125, "backtracks": 0.1013671875 }, "terminations": { "purchase": 4953, "generation_length": 167 } }, "success": { "n": 4471, "tasks": 601, "metrics": { "model_steps": { "n": 4471, "mean": 4.5587116976068, "median": 4.0, "p90": 5.0 }, "actions": { "n": 4471, "mean": 4.490494296577947, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 4471, "mean": 4.482442406620443, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 4471, "mean": 209.91438258118052, "median": 199.8, "p90": 277.0 }, "generated_tokens": { "n": 4471, "mean": 966.3795571460523, "median": 848.0, "p90": 1318.0 }, "first_turn_tokens": { "n": 4471, "mean": 173.14403936479536, "median": 167.0, "p90": 232.0 }, "searches": { "n": 4471, "mean": 1.072019682397674, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 4471, "mean": 1.0709013643480205, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 4471, "mean": 1.0648624468798926, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 4471, "mean": 1.0369044956385596, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 4471, "mean": 0.027957951241333035, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 4471, "mean": 0.07246700961753523, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 4471, "mean": 0.00782822634757325, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 4471, "mean": 1.2652650413777677, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 4471, "mean": 0.03377320509953031, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 4471, "mean": 0.06821740102885261, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 4471, "mean": 0.008051889957503914, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 4471, "mean": 274.4333979348191, "median": 258.0, "p90": 376.75 }, "first_product_rank": { "n": 4471, "mean": 1.1556698725117422, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 211.98523206751054, "token_complete_episodes": 4471, "rates": { "protocol_errors": 0.06150749273093268, "product_revisits": 0.02415566987251174, "option_changes": 0.023261015432789085, "backtracks": 0.053008275553567435 }, "terminations": { "purchase": 4471 } }, "failure": { "n": 649, "tasks": 159, "metrics": { "model_steps": { "n": 649, "mean": 8.969183359013867, "median": 5.0, "p90": 18.200000000000045 }, "actions": { "n": 649, "mean": 8.277349768875192, "median": 5.0, "p90": 17.0 }, "valid_actions": { "n": 649, "mean": 7.714946070878274, "median": 5.0, "p90": 15.0 }, "tokens_per_turn": { "n": 649, "mean": 299.439838826074, "median": 254.25, "p90": 419.7933333333334 }, "generated_tokens": { "n": 649, "mean": 2573.4530046224963, "median": 1713.0, "p90": 5332.6 }, "first_turn_tokens": { "n": 649, "mean": 174.98767334360554, "median": 172.0, "p90": 239.20000000000005 }, "searches": { "n": 649, "mean": 2.069337442218798, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 649, "mean": 2.0354391371340523, "median": 1.0, "p90": 5.0 }, "product_opens": { "n": 649, "mean": 1.7873651771956858, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 649, "mean": 1.300462249614792, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 649, "mean": 0.4869029275808937, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 649, "mean": 1.0847457627118644, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 649, "mean": 0.25577812018489987, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 649, "mean": 1.7750385208012327, "median": 1.0, "p90": 4.0 }, "option_changes": { "n": 649, "mean": 0.21417565485362094, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 649, "mean": 0.4345146379044684, "median": 0.0, "p90": 2.0 }, "invalid_actions": { "n": 649, "mean": 0.5624036979969184, "median": 0.0, "p90": 2.0 }, "reasoning_chars_per_turn": { "n": 649, "mean": 368.53223954873005, "median": 342.4, "p90": 537.7 }, "first_product_rank": { "n": 632, "mean": 1.6898734177215189, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.0, "reward_mean": 0.055706581554039174, "pooled_tokens_per_turn": 286.92166294451124, "token_complete_episodes": 649, "rates": { "protocol_errors": 0.28043143297380585, "product_revisits": 0.29275808936825887, "option_changes": 0.09861325115562404, "backtracks": 0.4345146379044684 }, "terminations": { "purchase": 482, "generation_length": 167 } }, "technical_n": 0, "skill_free": { "n": 992, "tasks": 124, "metrics": { "model_steps": { "n": 992, "mean": 5.178427419354839, "median": 4.0, "p90": 8.0 }, "actions": { "n": 992, "mean": 5.016129032258065, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 992, "mean": 4.963709677419355, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 992, "mean": 209.6747601202842, "median": 189.2840909090909, "p90": 278.475 }, "generated_tokens": { "n": 992, "mean": 1127.1532258064517, "median": 827.5, "p90": 2008.7999999999997 }, "first_turn_tokens": { "n": 992, "mean": 135.65625, "median": 135.0, "p90": 170.0 }, "searches": { "n": 992, "mean": 1.2137096774193548, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 992, "mean": 1.2137096774193548, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 992, "mean": 1.1491935483870968, "median": 1.0, "p90": 1.8999999999999773 }, "unique_products": { "n": 992, "mean": 1.0695564516129032, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 992, "mean": 0.07963709677419355, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 992, "mean": 0.21774193548387097, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 992, "mean": 0.10483870967741936, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 992, "mean": 1.315524193548387, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 992, "mean": 0.05443548387096774, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 992, "mean": 0.125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 992, "mean": 0.05241935483870968, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 992, "mean": 266.1163223190342, "median": 243.225, "p90": 376.13 }, "first_product_rank": { "n": 981, "mean": 1.3741080530071357, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.8679435483870968, "reward_mean": 0.8714717741935484, "pooled_tokens_per_turn": 217.6632275647265, "token_complete_episodes": 992, "rates": { "protocol_errors": 0.09879032258064516, "product_revisits": 0.06350806451612903, "option_changes": 0.03326612903225806, "backtracks": 0.11995967741935484 }, "terminations": { "purchase": 955, "generation_length": 37 } }, "assisted": { "n": 4128, "tasks": 516, "metrics": { "model_steps": { "n": 4128, "mean": 5.103197674418604, "median": 4.0, "p90": 6.0 }, "actions": { "n": 4128, "mean": 4.959544573643411, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 4128, "mean": 4.875, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 4128, "mean": 224.047068284704, "median": 208.5298245614035, "p90": 297.6500000000001 }, "generated_tokens": { "n": 4128, "mean": 1180.406492248062, "median": 888.0, "p90": 1832.3000000000002 }, "first_turn_tokens": { "n": 4128, "mean": 182.44258720930233, "median": 178.0, "p90": 239.0 }, "searches": { "n": 4128, "mean": 1.194767441860465, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 4128, "mean": 1.1882267441860466, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 4128, "mean": 1.158187984496124, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 4128, "mean": 1.0704941860465116, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 4128, "mean": 0.0876937984496124, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 4128, "mean": 0.19670542635658914, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 4128, "mean": 0.023498062015503876, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 4128, "mean": 1.3333333333333333, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 4128, "mean": 0.05717054263565891, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 4128, "mean": 0.1121608527131783, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 4128, "mean": 0.08454457364341085, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 4128, "mean": 291.22620007103194, "median": 272.225, "p90": 408.43 }, "first_product_rank": { "n": 4122, "mean": 1.185589519650655, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.874515503875969, "reward_mean": 0.8824257682724252, "pooled_tokens_per_turn": 231.30722491218077, "token_complete_episodes": 4128, "rates": { "protocol_errors": 0.0869670542635659, "product_revisits": 0.05692829457364341, "option_changes": 0.032703488372093026, "backtracks": 0.09689922480620156 }, "terminations": { "purchase": 3998, "generation_length": 130 } } } ], "steps": [ { "step": 1, "all": { "n": 247, "tasks": 32, "metrics": { "model_steps": { "n": 247, "mean": 5.8582995951417, "median": 4.0, "p90": 11.0 }, "actions": { "n": 247, "mean": 5.663967611336032, "median": 4.0, "p90": 9.400000000000006 }, "valid_actions": { "n": 247, "mean": 5.186234817813765, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 247, "mean": 333.4335499706787, "median": 322.75, "p90": 494.34000000000003 }, "generated_tokens": { "n": 247, "mean": 1799.9514170040486, "median": 1514.0, "p90": 2887.0000000000005 }, "first_turn_tokens": { "n": 247, "mean": 254.75708502024293, "median": 246.0, "p90": 360.80000000000007 }, "searches": { "n": 247, "mean": 1.2834008097165992, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 247, "mean": 1.2753036437246963, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 247, "mean": 1.2388663967611335, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 247, "mean": 1.1983805668016194, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 247, "mean": 0.04048582995951417, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 247, "mean": 0.2834008097165992, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 247, "mean": 0.09716599190283401, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 247, "mean": 1.3319838056680162, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 247, "mean": 0.05263157894736842, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 247, "mean": 0.145748987854251, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 247, "mean": 0.4777327935222672, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 247, "mean": 440.3670899757683, "median": 423.75, "p90": 677.15 }, "first_product_rank": { "n": 246, "mean": 4.764227642276423, "median": 4.0, "p90": 11.0 } }, "success_rate": 0.562753036437247, "reward_mean": 0.5843454790823212, "pooled_tokens_per_turn": 307.2480995162405, "token_complete_episodes": 247, "rates": { "protocol_errors": 0.08097165991902834, "product_revisits": 0.032388663967611336, "option_changes": 0.016194331983805668, "backtracks": 0.1417004048582996 }, "terminations": { "purchase": 235, "generation_length": 12 } }, "success": { "n": 139, "tasks": 29, "metrics": { "model_steps": { "n": 139, "mean": 4.726618705035971, "median": 4.0, "p90": 5.0 }, "actions": { "n": 139, "mean": 4.661870503597123, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 139, "mean": 4.654676258992806, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 139, "mean": 311.3235652626224, "median": 304.0, "p90": 436.45 }, "generated_tokens": { "n": 139, "mean": 1419.4604316546763, "median": 1334.0, "p90": 2033.2000000000003 }, "first_turn_tokens": { "n": 139, "mean": 256.8345323741007, "median": 251.0, "p90": 341.6 }, "searches": { "n": 139, "mean": 1.129496402877698, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 139, "mean": 1.129496402877698, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 139, "mean": 1.1079136690647482, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 139, "mean": 1.1079136690647482, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 139, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 139, "mean": 0.12949640287769784, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 139, "mean": 0.06474820143884892, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 139, "mean": 1.223021582733813, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 139, "mean": 0.007194244604316547, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 139, "mean": 0.06474820143884892, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 139, "mean": 0.007194244604316547, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 139, "mean": 407.40670991573114, "median": 386.75, "p90": 590.15 }, "first_product_rank": { "n": 139, "mean": 4.0071942446043165, "median": 2.0, "p90": 11.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 300.31202435312025, "token_complete_episodes": 139, "rates": { "protocol_errors": 0.04316546762589928, "product_revisits": 0.0, "option_changes": 0.007194244604316547, "backtracks": 0.07194244604316546 }, "terminations": { "purchase": 139 } }, "failure": { "n": 108, "tasks": 22, "metrics": { "model_steps": { "n": 108, "mean": 7.314814814814815, "median": 5.0, "p90": 16.599999999999994 }, "actions": { "n": 108, "mean": 6.953703703703703, "median": 5.0, "p90": 16.0 }, "valid_actions": { "n": 108, "mean": 5.87037037037037, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 108, "mean": 361.88991917826957, "median": 346.0777777777778, "p90": 546.875 }, "generated_tokens": { "n": 108, "mean": 2289.6574074074074, "median": 1851.5, "p90": 4161.499999999998 }, "first_turn_tokens": { "n": 108, "mean": 252.08333333333334, "median": 239.5, "p90": 370.99999999999994 }, "searches": { "n": 108, "mean": 1.4814814814814814, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 108, "mean": 1.462962962962963, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 108, "mean": 1.4074074074074074, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 108, "mean": 1.3148148148148149, "median": 1.0, "p90": 2.299999999999997 }, "product_revisits": { "n": 108, "mean": 0.09259259259259259, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 108, "mean": 0.48148148148148145, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 108, "mean": 0.1388888888888889, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 108, "mean": 1.4722222222222223, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 108, "mean": 0.1111111111111111, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 108, "mean": 0.25, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 108, "mean": 1.0833333333333333, "median": 0.0, "p90": 2.0 }, "reasoning_chars_per_turn": { "n": 108, "mean": 482.78831986785326, "median": 450.79230769230765, "p90": 751.7249999999999 }, "first_product_rank": { "n": 107, "mean": 5.747663551401869, "median": 4.0, "p90": 14.200000000000017 } }, "success_rate": 0.0, "reward_mean": 0.04938271604938271, "pooled_tokens_per_turn": 313.0164556962025, "token_complete_episodes": 108, "rates": { "protocol_errors": 0.12962962962962962, "product_revisits": 0.07407407407407407, "option_changes": 0.027777777777777776, "backtracks": 0.23148148148148148 }, "terminations": { "purchase": 96, "generation_length": 12 } } }, { "step": 2, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.3671875, "median": 4.0, "p90": 12.0 }, "actions": { "n": 256, "mean": 6.11328125, "median": 4.0, "p90": 10.5 }, "valid_actions": { "n": 256, "mean": 5.1796875, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 332.33622625738263, "median": 306.75, "p90": 466.0 }, "generated_tokens": { "n": 256, "mean": 1917.3828125, "median": 1434.5, "p90": 3914.0 }, "first_turn_tokens": { "n": 256, "mean": 247.26953125, "median": 238.0, "p90": 345.0 }, "searches": { "n": 256, "mean": 1.3125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.26171875, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.2578125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.1796875, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.3125, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.16796875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.21484375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.16796875, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.93359375, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 424.1980825872207, "median": 399.74285714285713, "p90": 610.25 }, "first_product_rank": { "n": 253, "mean": 5.877470355731226, "median": 2.0, "p90": 11.0 } }, "success_rate": 0.515625, "reward_mean": 0.5230034722222222, "pooled_tokens_per_turn": 301.13496932515335, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.10546875, "product_revisits": 0.0546875, "option_changes": 0.02734375, "backtracks": 0.16796875 }, "terminations": { "purchase": 234, "generation_length": 22 } }, "success": { "n": 132, "tasks": 24, "metrics": { "model_steps": { "n": 132, "mean": 4.386363636363637, "median": 4.0, "p90": 5.0 }, "actions": { "n": 132, "mean": 4.356060606060606, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 132, "mean": 4.325757575757576, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 132, "mean": 304.45319264069263, "median": 281.125, "p90": 411.55 }, "generated_tokens": { "n": 132, "mean": 1326.3484848484848, "median": 1215.0, "p90": 1830.7 }, "first_turn_tokens": { "n": 132, "mean": 246.49242424242425, "median": 234.0, "p90": 346.6 }, "searches": { "n": 132, "mean": 1.0303030303030303, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 132, "mean": 1.0303030303030303, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 132, "mean": 1.0303030303030303, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 132, "mean": 1.0303030303030303, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 132, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 132, "mean": 0.030303030303030304, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 132, "mean": 0.06818181818181818, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 132, "mean": 1.1666666666666667, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 132, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 132, "mean": 0.030303030303030304, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 132, "mean": 0.030303030303030304, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 132, "mean": 397.7906355218855, "median": 367.875, "p90": 551.5500000000001 }, "first_product_rank": { "n": 132, "mean": 4.196969696969697, "median": 1.0, "p90": 6.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 302.37996545768567, "token_complete_episodes": 132, "rates": { "protocol_errors": 0.030303030303030304, "product_revisits": 0.0, "option_changes": 0.0, "backtracks": 0.030303030303030304 }, "terminations": { "purchase": 132 } }, "failure": { "n": 124, "tasks": 22, "metrics": { "model_steps": { "n": 124, "mean": 8.475806451612904, "median": 4.5, "p90": 21.0 }, "actions": { "n": 124, "mean": 7.983870967741935, "median": 4.0, "p90": 19.0 }, "valid_actions": { "n": 124, "mean": 6.088709677419355, "median": 4.0, "p90": 12.0 }, "tokens_per_turn": { "n": 124, "mean": 362.0181652686977, "median": 331.2321428571429, "p90": 481.6511111111111 }, "generated_tokens": { "n": 124, "mean": 2546.548387096774, "median": 1893.5, "p90": 4941.8 }, "first_turn_tokens": { "n": 124, "mean": 248.09677419354838, "median": 239.0, "p90": 341.7 }, "searches": { "n": 124, "mean": 1.6129032258064515, "median": 1.0, "p90": 3.700000000000003 }, "unique_queries": { "n": 124, "mean": 1.5080645161290323, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 124, "mean": 1.5, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 124, "mean": 1.3387096774193548, "median": 1.0, "p90": 2.700000000000003 }, "product_revisits": { "n": 124, "mean": 0.16129032258064516, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 124, "mean": 0.6129032258064516, "median": 0.0, "p90": 2.700000000000003 }, "pagination": { "n": 124, "mean": 0.27419354838709675, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 124, "mean": 1.2661290322580645, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 124, "mean": 0.10483870967741936, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 124, "mean": 0.31451612903225806, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 124, "mean": 1.8951612903225807, "median": 0.0, "p90": 6.400000000000006 }, "reasoning_chars_per_turn": { "n": 124, "mean": 452.30923591483554, "median": 439.75, "p90": 645.4250000000001 }, "first_product_rank": { "n": 121, "mean": 7.710743801652892, "median": 3.0, "p90": 16.0 } }, "success_rate": 0.0, "reward_mean": 0.015232974910394265, "pooled_tokens_per_turn": 300.44909609895336, "token_complete_episodes": 124, "rates": { "protocol_errors": 0.18548387096774194, "product_revisits": 0.11290322580645161, "option_changes": 0.056451612903225805, "backtracks": 0.31451612903225806 }, "terminations": { "purchase": 102, "generation_length": 22 } } }, { "step": 3, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.8046875, "median": 4.0, "p90": 11.0 }, "actions": { "n": 256, "mean": 5.65625, "median": 4.0, "p90": 11.0 }, "valid_actions": { "n": 256, "mean": 5.1484375, "median": 4.0, "p90": 8.5 }, "tokens_per_turn": { "n": 256, "mean": 324.5652886535203, "median": 299.95, "p90": 467.7 }, "generated_tokens": { "n": 256, "mean": 1818.4453125, "median": 1385.5, "p90": 3403.0 }, "first_turn_tokens": { "n": 256, "mean": 238.11328125, "median": 227.0, "p90": 347.0 }, "searches": { "n": 256, "mean": 1.23828125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.234375, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.2109375, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.16796875, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.2421875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.1171875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.38671875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.1015625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.5078125, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 418.1866671811245, "median": 386.0, "p90": 610.5 }, "first_product_rank": { "n": 253, "mean": 4.8853754940711465, "median": 2.0, "p90": 15.0 } }, "success_rate": 0.5625, "reward_mean": 0.5738932291666666, "pooled_tokens_per_turn": 313.27187079407804, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.08203125, "product_revisits": 0.04296875, "option_changes": 0.015625, "backtracks": 0.14453125 }, "terminations": { "purchase": 244, "generation_length": 12 } }, "success": { "n": 144, "tasks": 26, "metrics": { "model_steps": { "n": 144, "mean": 4.625, "median": 4.0, "p90": 5.0 }, "actions": { "n": 144, "mean": 4.576388888888889, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 144, "mean": 4.548611111111111, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 144, "mean": 293.8019670915504, "median": 281.625, "p90": 407.67500000000007 }, "generated_tokens": { "n": 144, "mean": 1358.7083333333333, "median": 1222.5, "p90": 1922.3000000000004 }, "first_turn_tokens": { "n": 144, "mean": 221.36805555555554, "median": 216.0, "p90": 316.1 }, "searches": { "n": 144, "mean": 1.0694444444444444, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 144, "mean": 1.0694444444444444, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 144, "mean": 1.0694444444444444, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 144, "mean": 1.0486111111111112, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 144, "mean": 0.020833333333333332, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 144, "mean": 0.06944444444444445, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 144, "mean": 0.041666666666666664, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 144, "mean": 1.2986111111111112, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 144, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 144, "mean": 0.04861111111111111, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 144, "mean": 0.027777777777777776, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 144, "mean": 385.2064839866923, "median": 367.125, "p90": 544.525 }, "first_product_rank": { "n": 144, "mean": 3.3472222222222223, "median": 1.0, "p90": 8.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 293.77477477477476, "token_complete_episodes": 144, "rates": { "protocol_errors": 0.04861111111111111, "product_revisits": 0.020833333333333332, "option_changes": 0.0, "backtracks": 0.041666666666666664 }, "terminations": { "purchase": 144 } }, "failure": { "n": 112, "tasks": 22, "metrics": { "model_steps": { "n": 112, "mean": 7.321428571428571, "median": 5.0, "p90": 15.0 }, "actions": { "n": 112, "mean": 7.044642857142857, "median": 5.0, "p90": 13.0 }, "valid_actions": { "n": 112, "mean": 5.919642857142857, "median": 5.0, "p90": 10.0 }, "tokens_per_turn": { "n": 112, "mean": 364.11813066176734, "median": 337.625, "p90": 493.98 }, "generated_tokens": { "n": 112, "mean": 2409.535714285714, "median": 1938.5, "p90": 4512.7 }, "first_turn_tokens": { "n": 112, "mean": 259.64285714285717, "median": 255.5, "p90": 374.30000000000007 }, "searches": { "n": 112, "mean": 1.4553571428571428, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 112, "mean": 1.4464285714285714, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 112, "mean": 1.3928571428571428, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 112, "mean": 1.3214285714285714, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 112, "mean": 0.07142857142857142, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 112, "mean": 0.4642857142857143, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 112, "mean": 0.21428571428571427, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 112, "mean": 1.5, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 112, "mean": 0.08035714285714286, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 112, "mean": 0.16964285714285715, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 112, "mean": 1.125, "median": 0.0, "p90": 4.0 }, "reasoning_chars_per_turn": { "n": 112, "mean": 460.58975985968016, "median": 440.125, "p90": 652.61 }, "first_product_rank": { "n": 109, "mean": 6.91743119266055, "median": 3.0, "p90": 17.0 } }, "success_rate": 0.0, "reward_mean": 0.026041666666666668, "pooled_tokens_per_turn": 329.10731707317075, "token_complete_episodes": 112, "rates": { "protocol_errors": 0.125, "product_revisits": 0.07142857142857142, "option_changes": 0.03571428571428571, "backtracks": 0.2767857142857143 }, "terminations": { "purchase": 100, "generation_length": 12 } } }, { "step": 4, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.5234375, "median": 4.0, "p90": 8.0 }, "actions": { "n": 256, "mean": 5.4140625, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 4.9609375, "median": 4.0, "p90": 7.0 }, "tokens_per_turn": { "n": 256, "mean": 317.7995484097784, "median": 292.3611111111111, "p90": 444.75 }, "generated_tokens": { "n": 256, "mean": 1692.3984375, "median": 1352.5, "p90": 2852.0 }, "first_turn_tokens": { "n": 256, "mean": 239.3984375, "median": 237.0, "p90": 332.0 }, "searches": { "n": 256, "mean": 1.18359375, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.17578125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.1484375, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.140625, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.18359375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.15234375, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.33984375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0703125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.453125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 416.85104289489266, "median": 390.525, "p90": 577.7916666666667 }, "first_product_rank": { "n": 253, "mean": 5.051383399209486, "median": 1.0, "p90": 12.800000000000011 } }, "success_rate": 0.58984375, "reward_mean": 0.5943080357142857, "pooled_tokens_per_turn": 306.4031117397454, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.05859375, "product_revisits": 0.0078125, "option_changes": 0.015625, "backtracks": 0.125 }, "terminations": { "purchase": 244, "generation_length": 10, "action_limit": 2 } }, "success": { "n": 151, "tasks": 23, "metrics": { "model_steps": { "n": 151, "mean": 4.529801324503311, "median": 4.0, "p90": 6.0 }, "actions": { "n": 151, "mean": 4.503311258278146, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 151, "mean": 4.483443708609271, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 151, "mean": 306.95157284768214, "median": 288.5, "p90": 423.5 }, "generated_tokens": { "n": 151, "mean": 1385.635761589404, "median": 1222.0, "p90": 2080.0 }, "first_turn_tokens": { "n": 151, "mean": 248.36423841059602, "median": 240.0, "p90": 342.0 }, "searches": { "n": 151, "mean": 1.0596026490066226, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 151, "mean": 1.0596026490066226, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 151, "mean": 1.0596026490066226, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 151, "mean": 1.0596026490066226, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 151, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 151, "mean": 0.059602649006622516, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 151, "mean": 0.013245033112582781, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 151, "mean": 1.2913907284768211, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 151, "mean": 0.033112582781456956, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 151, "mean": 0.026490066225165563, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 151, "mean": 0.019867549668874173, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 151, "mean": 401.4774624198465, "median": 386.5, "p90": 566.5 }, "first_product_rank": { "n": 151, "mean": 4.046357615894039, "median": 1.0, "p90": 11.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 305.8932748538012, "token_complete_episodes": 151, "rates": { "protocol_errors": 0.026490066225165563, "product_revisits": 0.0, "option_changes": 0.019867549668874173, "backtracks": 0.052980132450331126 }, "terminations": { "purchase": 151 } }, "failure": { "n": 105, "tasks": 20, "metrics": { "model_steps": { "n": 105, "mean": 6.9523809523809526, "median": 5.0, "p90": 12.600000000000009 }, "actions": { "n": 105, "mean": 6.723809523809524, "median": 5.0, "p90": 12.0 }, "valid_actions": { "n": 105, "mean": 5.647619047619048, "median": 5.0, "p90": 9.600000000000009 }, "tokens_per_turn": { "n": 105, "mean": 333.39997040860266, "median": 306.0, "p90": 497.8500000000002 }, "generated_tokens": { "n": 105, "mean": 2133.552380952381, "median": 1681.0, "p90": 3910.4000000000033 }, "first_turn_tokens": { "n": 105, "mean": 226.5047619047619, "median": 227.0, "p90": 307.6 }, "searches": { "n": 105, "mean": 1.361904761904762, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 105, "mean": 1.3428571428571427, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 105, "mean": 1.276190476190476, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 105, "mean": 1.2571428571428571, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 105, "mean": 0.01904761904761905, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 105, "mean": 0.3619047619047619, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 105, "mean": 0.3523809523809524, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 105, "mean": 1.4095238095238096, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 105, "mean": 0.01904761904761905, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 105, "mean": 0.13333333333333333, "median": 0.0, "p90": 0.6000000000000085 }, "invalid_actions": { "n": 105, "mean": 1.0761904761904761, "median": 0.0, "p90": 2.0 }, "reasoning_chars_per_turn": { "n": 105, "mean": 438.95971576853043, "median": 406.0, "p90": 656.0100000000001 }, "first_product_rank": { "n": 102, "mean": 6.53921568627451, "median": 2.0, "p90": 15.0 } }, "success_rate": 0.0, "reward_mean": 0.010884353741496598, "pooled_tokens_per_turn": 306.8808219178082, "token_complete_episodes": 105, "rates": { "protocol_errors": 0.10476190476190476, "product_revisits": 0.01904761904761905, "option_changes": 0.009523809523809525, "backtracks": 0.22857142857142856 }, "terminations": { "purchase": 93, "generation_length": 10, "action_limit": 2 } } }, { "step": 5, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.34375, "median": 4.0, "p90": 11.0 }, "actions": { "n": 256, "mean": 6.1328125, "median": 4.0, "p90": 10.5 }, "valid_actions": { "n": 256, "mean": 5.62109375, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 256, "mean": 314.9353490866301, "median": 288.35, "p90": 431.5 }, "generated_tokens": { "n": 256, "mean": 1863.3203125, "median": 1372.0, "p90": 3571.0 }, "first_turn_tokens": { "n": 256, "mean": 227.09765625, "median": 221.5, "p90": 334.5 }, "searches": { "n": 256, "mean": 1.3984375, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.37890625, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.32421875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.30078125, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.3984375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.20703125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.37109375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.1015625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.13671875, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.51171875, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 386.1422886626701, "median": 366.43055555555554, "p90": 550.75 }, "first_product_rank": { "n": 248, "mean": 4.463709677419355, "median": 2.0, "p90": 11.0 } }, "success_rate": 0.46875, "reward_mean": 0.4833984375, "pooled_tokens_per_turn": 293.72536945812806, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.11328125, "product_revisits": 0.0234375, "option_changes": 0.04296875, "backtracks": 0.21875 }, "terminations": { "purchase": 236, "generation_length": 19, "action_limit": 1 } }, "success": { "n": 120, "tasks": 22, "metrics": { "model_steps": { "n": 120, "mean": 5.025, "median": 4.0, "p90": 8.0 }, "actions": { "n": 120, "mean": 4.975, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 120, "mean": 4.941666666666666, "median": 4.0, "p90": 7.1000000000000085 }, "tokens_per_turn": { "n": 120, "mean": 281.6592174723425, "median": 272.175, "p90": 390.5 }, "generated_tokens": { "n": 120, "mean": 1367.2166666666667, "median": 1175.5, "p90": 2040.0000000000005 }, "first_turn_tokens": { "n": 120, "mean": 229.21666666666667, "median": 223.0, "p90": 340.20000000000005 }, "searches": { "n": 120, "mean": 1.2166666666666666, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 120, "mean": 1.2083333333333333, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 120, "mean": 1.2166666666666666, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 120, "mean": 1.2166666666666666, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 120, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 120, "mean": 0.21666666666666667, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 120, "mean": 0.016666666666666666, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 120, "mean": 1.275, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 120, "mean": 0.008333333333333333, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 120, "mean": 0.05, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 120, "mean": 0.03333333333333333, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 120, "mean": 358.45990349927854, "median": 337.875, "p90": 484.22500000000036 }, "first_product_rank": { "n": 120, "mean": 4.108333333333333, "median": 1.5, "p90": 11.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 272.08291873963515, "token_complete_episodes": 120, "rates": { "protocol_errors": 0.03333333333333333, "product_revisits": 0.0, "option_changes": 0.008333333333333333, "backtracks": 0.11666666666666667 }, "terminations": { "purchase": 120 } }, "failure": { "n": 136, "tasks": 25, "metrics": { "model_steps": { "n": 136, "mean": 7.507352941176471, "median": 5.0, "p90": 14.0 }, "actions": { "n": 136, "mean": 7.154411764705882, "median": 5.0, "p90": 13.0 }, "valid_actions": { "n": 136, "mean": 6.220588235294118, "median": 4.0, "p90": 11.5 }, "tokens_per_turn": { "n": 136, "mean": 344.29664168747206, "median": 313.8125, "p90": 438.125 }, "generated_tokens": { "n": 136, "mean": 2301.0588235294117, "median": 1720.0, "p90": 4228.5 }, "first_turn_tokens": { "n": 136, "mean": 225.22794117647058, "median": 220.0, "p90": 326.0 }, "searches": { "n": 136, "mean": 1.5588235294117647, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 136, "mean": 1.5294117647058822, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 136, "mean": 1.4191176470588236, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 136, "mean": 1.375, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 136, "mean": 0.04411764705882353, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 136, "mean": 0.5588235294117647, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 136, "mean": 0.375, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 136, "mean": 1.4558823529411764, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 136, "mean": 0.18382352941176472, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 136, "mean": 0.21323529411764705, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 136, "mean": 0.9338235294117647, "median": 0.0, "p90": 3.0 }, "reasoning_chars_per_turn": { "n": 136, "mean": 410.5679226303686, "median": 406.30357142857144, "p90": 557.275 }, "first_product_rank": { "n": 128, "mean": 4.796875, "median": 2.5, "p90": 11.0 } }, "success_rate": 0.0, "reward_mean": 0.027573529411764705, "pooled_tokens_per_turn": 306.50734573947113, "token_complete_episodes": 136, "rates": { "protocol_errors": 0.18382352941176472, "product_revisits": 0.04411764705882353, "option_changes": 0.07352941176470588, "backtracks": 0.3088235294117647 }, "terminations": { "purchase": 116, "generation_length": 19, "action_limit": 1 } } }, { "step": 6, "all": { "n": 214, "tasks": 28, "metrics": { "model_steps": { "n": 214, "mean": 4.780373831775701, "median": 4.0, "p90": 5.0 }, "actions": { "n": 214, "mean": 4.742990654205608, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 214, "mean": 4.663551401869159, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 214, "mean": 294.0064608029561, "median": 278.2678571428571, "p90": 410.70000000000005 }, "generated_tokens": { "n": 214, "mean": 1391.3317757009345, "median": 1252.0, "p90": 1865.0 }, "first_turn_tokens": { "n": 214, "mean": 221.20560747663552, "median": 213.0, "p90": 322.0 }, "searches": { "n": 214, "mean": 1.102803738317757, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 214, "mean": 1.102803738317757, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 214, "mean": 1.0654205607476634, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 214, "mean": 1.0560747663551402, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 214, "mean": 0.009345794392523364, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 214, "mean": 0.102803738317757, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 214, "mean": 0.1261682242990654, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 214, "mean": 1.2897196261682242, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 214, "mean": 0.014018691588785047, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 214, "mean": 0.014018691588785047, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 214, "mean": 0.0794392523364486, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 214, "mean": 381.22931511177114, "median": 357.575, "p90": 536.2500000000001 }, "first_product_rank": { "n": 211, "mean": 4.819905213270142, "median": 2.0, "p90": 12.0 } }, "success_rate": 0.6074766355140186, "reward_mean": 0.6259735202492211, "pooled_tokens_per_turn": 291.0508308895406, "token_complete_episodes": 214, "rates": { "protocol_errors": 0.014018691588785047, "product_revisits": 0.009345794392523364, "option_changes": 0.014018691588785047, "backtracks": 0.07476635514018691 }, "terminations": { "purchase": 209, "generation_length": 5 } }, "success": { "n": 130, "tasks": 25, "metrics": { "model_steps": { "n": 130, "mean": 4.461538461538462, "median": 4.0, "p90": 5.0 }, "actions": { "n": 130, "mean": 4.461538461538462, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 130, "mean": 4.446153846153846, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 130, "mean": 272.4086721611722, "median": 258.875, "p90": 377.1 }, "generated_tokens": { "n": 130, "mean": 1201.2692307692307, "median": 1135.0, "p90": 1585.1000000000006 }, "first_turn_tokens": { "n": 130, "mean": 217.93846153846152, "median": 212.0, "p90": 305.1 }, "searches": { "n": 130, "mean": 1.0384615384615385, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 130, "mean": 1.0384615384615385, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 130, "mean": 1.0384615384615385, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 130, "mean": 1.0384615384615385, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 130, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 130, "mean": 0.038461538461538464, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 130, "mean": 0.015384615384615385, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 130, "mean": 1.3153846153846154, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 130, "mean": 0.007692307692307693, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 130, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 130, "mean": 0.015384615384615385, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 130, "mean": 349.35093406593404, "median": 333.5, "p90": 509.825 }, "first_product_rank": { "n": 130, "mean": 3.853846153846154, "median": 2.0, "p90": 9.300000000000026 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 269.25, "token_complete_episodes": 130, "rates": { "protocol_errors": 0.0, "product_revisits": 0.0, "option_changes": 0.007692307692307693, "backtracks": 0.03076923076923077 }, "terminations": { "purchase": 130 } }, "failure": { "n": 84, "tasks": 22, "metrics": { "model_steps": { "n": 84, "mean": 5.273809523809524, "median": 4.0, "p90": 9.0 }, "actions": { "n": 84, "mean": 5.178571428571429, "median": 4.0, "p90": 9.0 }, "valid_actions": { "n": 84, "mean": 5.0, "median": 4.0, "p90": 8.700000000000003 }, "tokens_per_turn": { "n": 84, "mean": 327.4316098914313, "median": 319.53125, "p90": 433.77500000000003 }, "generated_tokens": { "n": 84, "mean": 1685.4761904761904, "median": 1408.5, "p90": 2398.3 }, "first_turn_tokens": { "n": 84, "mean": 226.26190476190476, "median": 221.0, "p90": 331.90000000000003 }, "searches": { "n": 84, "mean": 1.2023809523809523, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 84, "mean": 1.2023809523809523, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 84, "mean": 1.1071428571428572, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 84, "mean": 1.0833333333333333, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 84, "mean": 0.023809523809523808, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 84, "mean": 0.20238095238095238, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 84, "mean": 0.2976190476190476, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 84, "mean": 1.25, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 84, "mean": 0.023809523809523808, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 84, "mean": 0.03571428571428571, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 84, "mean": 0.17857142857142858, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 84, "mean": 430.5649048255666, "median": 413.9, "p90": 561.9250000000001 }, "first_product_rank": { "n": 81, "mean": 6.37037037037037, "median": 5.0, "p90": 18.0 } }, "success_rate": 0.0, "reward_mean": 0.04712301587301587, "pooled_tokens_per_turn": 319.59367945823925, "token_complete_episodes": 84, "rates": { "protocol_errors": 0.03571428571428571, "product_revisits": 0.023809523809523808, "option_changes": 0.023809523809523808, "backtracks": 0.14285714285714285 }, "terminations": { "purchase": 79, "generation_length": 5 } } }, { "step": 7, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.47265625, "median": 4.0, "p90": 9.0 }, "actions": { "n": 256, "mean": 5.4140625, "median": 4.0, "p90": 8.5 }, "valid_actions": { "n": 256, "mean": 5.16015625, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 262.9712850846826, "median": 251.875, "p90": 364.0 }, "generated_tokens": { "n": 256, "mean": 1411.71484375, "median": 1110.0, "p90": 2276.0 }, "first_turn_tokens": { "n": 256, "mean": 206.2421875, "median": 198.5, "p90": 293.0 }, "searches": { "n": 256, "mean": 1.1328125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.1328125, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.125, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.078125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.1328125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.109375, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.6796875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.36328125, "median": 0.0, "p90": 0.5 }, "protocol_errors": { "n": 256, "mean": 0.0390625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.25390625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 339.704418203446, "median": 324.375, "p90": 475.35 }, "first_product_rank": { "n": 256, "mean": 5.50390625, "median": 1.0, "p90": 17.0 } }, "success_rate": 0.71875, "reward_mean": 0.72265625, "pooled_tokens_per_turn": 257.95788722341183, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.03125, "product_revisits": 0.04296875, "option_changes": 0.1015625, "backtracks": 0.08984375 }, "terminations": { "purchase": 251, "generation_length": 5 } }, "success": { "n": 184, "tasks": 28, "metrics": { "model_steps": { "n": 184, "mean": 4.728260869565218, "median": 4.0, "p90": 5.0 }, "actions": { "n": 184, "mean": 4.706521739130435, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 184, "mean": 4.690217391304348, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 184, "mean": 257.45946342305035, "median": 237.5, "p90": 358.09500000000014 }, "generated_tokens": { "n": 184, "mean": 1214.9510869565217, "median": 1039.5, "p90": 1928.5000000000007 }, "first_turn_tokens": { "n": 184, "mean": 204.22826086956522, "median": 196.5, "p90": 293.0 }, "searches": { "n": 184, "mean": 1.059782608695652, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 184, "mean": 1.059782608695652, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 184, "mean": 1.059782608695652, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 184, "mean": 1.0326086956521738, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 184, "mean": 0.02717391304347826, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 184, "mean": 0.059782608695652176, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 184, "mean": 0.14130434782608695, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 184, "mean": 1.3695652173913044, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 184, "mean": 0.03804347826086957, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 184, "mean": 0.021739130434782608, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 184, "mean": 0.016304347826086956, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 184, "mean": 332.7534269323672, "median": 321.8, "p90": 462.925 }, "first_product_rank": { "n": 184, "mean": 5.782608695652174, "median": 1.0, "p90": 17.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 256.9551724137931, "token_complete_episodes": 184, "rates": { "protocol_errors": 0.016304347826086956, "product_revisits": 0.02717391304347826, "option_changes": 0.03804347826086957, "backtracks": 0.04891304347826087 }, "terminations": { "purchase": 184 } }, "failure": { "n": 72, "tasks": 20, "metrics": { "model_steps": { "n": 72, "mean": 7.375, "median": 4.0, "p90": 18.699999999999996 }, "actions": { "n": 72, "mean": 7.222222222222222, "median": 4.0, "p90": 16.9 }, "valid_actions": { "n": 72, "mean": 6.361111111111111, "median": 4.0, "p90": 12.0 }, "tokens_per_turn": { "n": 72, "mean": 277.05705155329844, "median": 261.55555555555554, "p90": 388.53999999999996 }, "generated_tokens": { "n": 72, "mean": 1914.5555555555557, "median": 1330.0, "p90": 3679.8 }, "first_turn_tokens": { "n": 72, "mean": 211.38888888888889, "median": 200.0, "p90": 292.4 }, "searches": { "n": 72, "mean": 1.3194444444444444, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 72, "mean": 1.3194444444444444, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 72, "mean": 1.2916666666666667, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 72, "mean": 1.1944444444444444, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 72, "mean": 0.09722222222222222, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 72, "mean": 0.3194444444444444, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 72, "mean": 0.027777777777777776, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 72, "mean": 2.4722222222222223, "median": 1.0, "p90": 6.0 }, "option_changes": { "n": 72, "mean": 1.1944444444444444, "median": 0.0, "p90": 4.0 }, "protocol_errors": { "n": 72, "mean": 0.08333333333333333, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 72, "mean": 0.8611111111111112, "median": 0.0, "p90": 3.8999999999999986 }, "reasoning_chars_per_turn": { "n": 72, "mean": 357.4680625628697, "median": 345.25, "p90": 479.5 }, "first_product_rank": { "n": 72, "mean": 4.791666666666667, "median": 2.0, "p90": 15.499999999999993 } }, "success_rate": 0.0, "reward_mean": 0.013888888888888888, "pooled_tokens_per_turn": 259.60075329566854, "token_complete_episodes": 72, "rates": { "protocol_errors": 0.06944444444444445, "product_revisits": 0.08333333333333333, "option_changes": 0.2638888888888889, "backtracks": 0.19444444444444445 }, "terminations": { "purchase": 67, "generation_length": 5 } } }, { "step": 8, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.9453125, "median": 4.0, "p90": 12.0 }, "actions": { "n": 256, "mean": 5.70703125, "median": 4.0, "p90": 11.0 }, "valid_actions": { "n": 256, "mean": 5.54296875, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 256, "mean": 263.7311747584346, "median": 248.0, "p90": 387.25 }, "generated_tokens": { "n": 256, "mean": 1457.69921875, "median": 1213.5, "p90": 2420.0 }, "first_turn_tokens": { "n": 256, "mean": 183.5, "median": 175.0, "p90": 256.0 }, "searches": { "n": 256, "mean": 1.28515625, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.2734375, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.16015625, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.10546875, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.0546875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.28515625, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.55859375, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 256, "mean": 1.359375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.08203125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.1328125, "median": 0.0, "p90": 0.5 }, "invalid_actions": { "n": 256, "mean": 0.1640625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 338.70672485812776, "median": 315.875, "p90": 499.25 }, "first_product_rank": { "n": 245, "mean": 6.66530612244898, "median": 2.0, "p90": 16.0 } }, "success_rate": 0.57421875, "reward_mean": 0.57421875, "pooled_tokens_per_turn": 245.18462549277265, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.1015625, "product_revisits": 0.0390625, "option_changes": 0.03515625, "backtracks": 0.16796875 }, "terminations": { "purchase": 229, "generation_length": 27 } }, "success": { "n": 147, "tasks": 27, "metrics": { "model_steps": { "n": 147, "mean": 4.605442176870748, "median": 4.0, "p90": 5.0 }, "actions": { "n": 147, "mean": 4.571428571428571, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 147, "mean": 4.5646258503401365, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 147, "mean": 248.41100178166505, "median": 225.75, "p90": 348.90000000000003 }, "generated_tokens": { "n": 147, "mean": 1141.0884353741496, "median": 991.0, "p90": 1776.2 }, "first_turn_tokens": { "n": 147, "mean": 184.7687074829932, "median": 176.0, "p90": 258.8 }, "searches": { "n": 147, "mean": 1.08843537414966, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 147, "mean": 1.08843537414966, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 147, "mean": 1.0748299319727892, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 147, "mean": 1.054421768707483, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 147, "mean": 0.02040816326530612, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 147, "mean": 0.08843537414965986, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 147, "mean": 0.08163265306122448, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 147, "mean": 1.2312925170068028, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 147, "mean": 0.07482993197278912, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 147, "mean": 0.034013605442176874, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 147, "mean": 0.006802721088435374, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 147, "mean": 316.3681567865242, "median": 287.75, "p90": 445.27000000000004 }, "first_product_rank": { "n": 147, "mean": 4.0, "median": 1.0, "p90": 6.800000000000011 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 247.76957163958642, "token_complete_episodes": 147, "rates": { "protocol_errors": 0.027210884353741496, "product_revisits": 0.013605442176870748, "option_changes": 0.04081632653061224, "backtracks": 0.05442176870748299 }, "terminations": { "purchase": 147 } }, "failure": { "n": 109, "tasks": 20, "metrics": { "model_steps": { "n": 109, "mean": 7.752293577981652, "median": 5.0, "p90": 15.0 }, "actions": { "n": 109, "mean": 7.238532110091743, "median": 5.0, "p90": 13.0 }, "valid_actions": { "n": 109, "mean": 6.862385321100917, "median": 5.0, "p90": 11.200000000000003 }, "tokens_per_turn": { "n": 109, "mean": 284.3923254702247, "median": 276.75, "p90": 413.55 }, "generated_tokens": { "n": 109, "mean": 1884.6880733944954, "median": 1525.0, "p90": 3327.600000000001 }, "first_turn_tokens": { "n": 109, "mean": 181.78899082568807, "median": 174.0, "p90": 247.4 }, "searches": { "n": 109, "mean": 1.5504587155963303, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 109, "mean": 1.5229357798165137, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 109, "mean": 1.275229357798165, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 109, "mean": 1.1743119266055047, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 109, "mean": 0.10091743119266056, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 109, "mean": 0.5504587155963303, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 109, "mean": 1.201834862385321, "median": 0.0, "p90": 7.0 }, "option_selections": { "n": 109, "mean": 1.5321100917431192, "median": 1.0, "p90": 2.200000000000003 }, "option_changes": { "n": 109, "mean": 0.09174311926605505, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 109, "mean": 0.26605504587155965, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 109, "mean": 0.3761467889908257, "median": 0.0, "p90": 0.20000000000000284 }, "reasoning_chars_per_turn": { "n": 109, "mean": 368.8330506060703, "median": 365.5, "p90": 545.8 }, "first_product_rank": { "n": 98, "mean": 10.66326530612245, "median": 4.0, "p90": 20.0 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 243.11360946745563, "token_complete_episodes": 109, "rates": { "protocol_errors": 0.2018348623853211, "product_revisits": 0.07339449541284404, "option_changes": 0.027522935779816515, "backtracks": 0.3211009174311927 }, "terminations": { "purchase": 82, "generation_length": 27 } } }, { "step": 9, "all": { "n": 255, "tasks": 32, "metrics": { "model_steps": { "n": 255, "mean": 5.545098039215686, "median": 4.0, "p90": 8.0 }, "actions": { "n": 255, "mean": 5.435294117647059, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 255, "mean": 5.145098039215687, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 255, "mean": 294.9914881306058, "median": 278.7142857142857, "p90": 433.55 }, "generated_tokens": { "n": 255, "mean": 1545.2039215686275, "median": 1254.0, "p90": 2497.599999999999 }, "first_turn_tokens": { "n": 255, "mean": 207.45098039215685, "median": 202.0, "p90": 290.0 }, "searches": { "n": 255, "mean": 1.203921568627451, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 255, "mean": 1.184313725490196, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 255, "mean": 1.1607843137254903, "median": 1.0, "p90": 1.5999999999999943 }, "unique_products": { "n": 255, "mean": 1.0941176470588236, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 255, "mean": 0.06666666666666667, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 255, "mean": 0.20784313725490197, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 255, "mean": 0.12549019607843137, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 255, "mean": 1.4862745098039216, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 255, "mean": 0.08627450980392157, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 255, "mean": 0.07058823529411765, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 255, "mean": 0.2901960784313726, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 255, "mean": 378.4325276575276, "median": 359.5, "p90": 549.05 }, "first_product_rank": { "n": 252, "mean": 4.007936507936508, "median": 1.0, "p90": 11.0 } }, "success_rate": 0.611764705882353, "reward_mean": 0.6154901960784314, "pooled_tokens_per_turn": 278.6612446958982, "token_complete_episodes": 255, "rates": { "protocol_errors": 0.054901960784313725, "product_revisits": 0.027450980392156862, "option_changes": 0.023529411764705882, "backtracks": 0.11372549019607843 }, "terminations": { "purchase": 245, "generation_length": 10 } }, "success": { "n": 156, "tasks": 27, "metrics": { "model_steps": { "n": 156, "mean": 4.551282051282051, "median": 4.0, "p90": 5.0 }, "actions": { "n": 156, "mean": 4.5256410256410255, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 156, "mean": 4.512820512820513, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 156, "mean": 282.01145474895475, "median": 273.25, "p90": 403.5 }, "generated_tokens": { "n": 156, "mean": 1266.2564102564102, "median": 1132.0, "p90": 1865.0 }, "first_turn_tokens": { "n": 156, "mean": 210.71153846153845, "median": 205.5, "p90": 295.0 }, "searches": { "n": 156, "mean": 1.0512820512820513, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 156, "mean": 1.0512820512820513, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 156, "mean": 1.0512820512820513, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 156, "mean": 1.0512820512820513, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 156, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 156, "mean": 0.05128205128205128, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 156, "mean": 0.05128205128205128, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 156, "mean": 1.3076923076923077, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 156, "mean": 0.08974358974358974, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 156, "mean": 0.02564102564102564, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 156, "mean": 0.01282051282051282, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 156, "mean": 362.59384435009434, "median": 352.5, "p90": 522.75 }, "first_product_rank": { "n": 156, "mean": 3.7756410256410255, "median": 1.0, "p90": 10.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 278.21971830985916, "token_complete_episodes": 156, "rates": { "protocol_errors": 0.02564102564102564, "product_revisits": 0.0, "option_changes": 0.02564102564102564, "backtracks": 0.05128205128205128 }, "terminations": { "purchase": 156 } }, "failure": { "n": 99, "tasks": 21, "metrics": { "model_steps": { "n": 99, "mean": 7.111111111111111, "median": 4.0, "p90": 12.0 }, "actions": { "n": 99, "mean": 6.8686868686868685, "median": 4.0, "p90": 11.0 }, "valid_actions": { "n": 99, "mean": 6.141414141414141, "median": 4.0, "p90": 10.200000000000003 }, "tokens_per_turn": { "n": 99, "mean": 315.44487406532863, "median": 298.25, "p90": 471.8 }, "generated_tokens": { "n": 99, "mean": 1984.7575757575758, "median": 1425.0, "p90": 4100.400000000001 }, "first_turn_tokens": { "n": 99, "mean": 202.31313131313132, "median": 196.0, "p90": 287.4 }, "searches": { "n": 99, "mean": 1.4444444444444444, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 99, "mean": 1.393939393939394, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 99, "mean": 1.3333333333333333, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 99, "mean": 1.1616161616161615, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 99, "mean": 0.1717171717171717, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 99, "mean": 0.45454545454545453, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 99, "mean": 0.24242424242424243, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 99, "mean": 1.7676767676767677, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 99, "mean": 0.08080808080808081, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 99, "mean": 0.1414141414141414, "median": 0.0, "p90": 0.20000000000000284 }, "invalid_actions": { "n": 99, "mean": 0.7272727272727273, "median": 0.0, "p90": 1.2000000000000028 }, "reasoning_chars_per_turn": { "n": 99, "mean": 403.39045286924073, "median": 385.0, "p90": 618.08 }, "first_product_rank": { "n": 96, "mean": 4.385416666666667, "median": 2.0, "p90": 16.0 } }, "success_rate": 0.0, "reward_mean": 0.009595959595959595, "pooled_tokens_per_turn": 279.10653409090907, "token_complete_episodes": 99, "rates": { "protocol_errors": 0.10101010101010101, "product_revisits": 0.0707070707070707, "option_changes": 0.020202020202020204, "backtracks": 0.21212121212121213 }, "terminations": { "purchase": 89, "generation_length": 10 } } }, { "step": 10, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.9453125, "median": 4.0, "p90": 6.5 }, "actions": { "n": 256, "mean": 4.8828125, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 256, "mean": 4.81640625, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 256, "mean": 282.78280136746935, "median": 262.95, "p90": 365.25 }, "generated_tokens": { "n": 256, "mean": 1364.7890625, "median": 1205.0, "p90": 2126.0 }, "first_turn_tokens": { "n": 256, "mean": 198.25, "median": 190.0, "p90": 274.0 }, "searches": { "n": 256, "mean": 1.11328125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.109375, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.09765625, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0859375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.11328125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0390625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.46484375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0703125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.06640625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 350.48082992311504, "median": 337.6, "p90": 476.52777777777777 }, "first_product_rank": { "n": 254, "mean": 4.188976377952756, "median": 1.0, "p90": 11.700000000000017 } }, "success_rate": 0.6328125, "reward_mean": 0.6733668154761905, "pooled_tokens_per_turn": 275.97630331753555, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.04296875, "product_revisits": 0.01171875, "option_changes": 0.03515625, "backtracks": 0.08984375 }, "terminations": { "purchase": 253, "generation_length": 3 } }, "success": { "n": 162, "tasks": 26, "metrics": { "model_steps": { "n": 162, "mean": 4.6419753086419755, "median": 4.0, "p90": 5.0 }, "actions": { "n": 162, "mean": 4.604938271604938, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 162, "mean": 4.58641975308642, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 162, "mean": 277.89987952719434, "median": 261.9, "p90": 365.5 }, "generated_tokens": { "n": 162, "mean": 1272.7654320987654, "median": 1146.5, "p90": 1751.3000000000002 }, "first_turn_tokens": { "n": 162, "mean": 202.2469135802469, "median": 199.5, "p90": 276.8 }, "searches": { "n": 162, "mean": 1.0864197530864197, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 162, "mean": 1.0802469135802468, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 162, "mean": 1.0740740740740742, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 162, "mean": 1.0617283950617284, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 162, "mean": 0.012345679012345678, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 162, "mean": 0.08641975308641975, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 162, "mean": 0.018518518518518517, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 162, "mean": 1.3209876543209877, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 162, "mean": 0.018518518518518517, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 162, "mean": 0.037037037037037035, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 162, "mean": 0.018518518518518517, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 162, "mean": 352.8695742700372, "median": 335.625, "p90": 465.495 }, "first_product_rank": { "n": 162, "mean": 3.7160493827160495, "median": 1.0, "p90": 11.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 274.18617021276594, "token_complete_episodes": 162, "rates": { "protocol_errors": 0.037037037037037035, "product_revisits": 0.012345679012345678, "option_changes": 0.012345679012345678, "backtracks": 0.08024691358024691 }, "terminations": { "purchase": 162 } }, "failure": { "n": 94, "tasks": 19, "metrics": { "model_steps": { "n": 94, "mean": 5.468085106382978, "median": 5.0, "p90": 7.700000000000003 }, "actions": { "n": 94, "mean": 5.361702127659575, "median": 5.0, "p90": 7.700000000000003 }, "valid_actions": { "n": 94, "mean": 5.212765957446808, "median": 5.0, "p90": 7.0 }, "tokens_per_turn": { "n": 94, "mean": 291.19804964539003, "median": 264.25, "p90": 364.05833333333334 }, "generated_tokens": { "n": 94, "mean": 1523.3829787234042, "median": 1247.5, "p90": 2953.6000000000004 }, "first_turn_tokens": { "n": 94, "mean": 191.36170212765958, "median": 179.5, "p90": 259.1 }, "searches": { "n": 94, "mean": 1.1595744680851063, "median": 1.0, "p90": 1.7000000000000028 }, "unique_queries": { "n": 94, "mean": 1.1595744680851063, "median": 1.0, "p90": 1.7000000000000028 }, "product_opens": { "n": 94, "mean": 1.1382978723404256, "median": 1.0, "p90": 1.7000000000000028 }, "unique_products": { "n": 94, "mean": 1.127659574468085, "median": 1.0, "p90": 1.7000000000000028 }, "product_revisits": { "n": 94, "mean": 0.010638297872340425, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 94, "mean": 0.1595744680851064, "median": 0.0, "p90": 0.7000000000000028 }, "pagination": { "n": 94, "mean": 0.07446808510638298, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 94, "mean": 1.7127659574468086, "median": 2.0, "p90": 2.0 }, "option_changes": { "n": 94, "mean": 0.1595744680851064, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 94, "mean": 0.07446808510638298, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 94, "mean": 0.14893617021276595, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 94, "mean": 346.36405775075986, "median": 337.6, "p90": 477.3388888888889 }, "first_product_rank": { "n": 92, "mean": 5.021739130434782, "median": 2.0, "p90": 12.0 } }, "success_rate": 0.0, "reward_mean": 0.11044579533941234, "pooled_tokens_per_turn": 278.5953307392996, "token_complete_episodes": 94, "rates": { "protocol_errors": 0.05319148936170213, "product_revisits": 0.010638297872340425, "option_changes": 0.07446808510638298, "backtracks": 0.10638297872340426 }, "terminations": { "purchase": 91, "generation_length": 3 } } }, { "step": 11, "all": { "n": 255, "tasks": 32, "metrics": { "model_steps": { "n": 255, "mean": 5.905882352941177, "median": 4.0, "p90": 11.0 }, "actions": { "n": 255, "mean": 5.713725490196079, "median": 4.0, "p90": 10.0 }, "valid_actions": { "n": 255, "mean": 5.6156862745098035, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 255, "mean": 240.2189176973914, "median": 225.0, "p90": 308.3 }, "generated_tokens": { "n": 255, "mean": 1394.929411764706, "median": 1030.0, "p90": 2611.8 }, "first_turn_tokens": { "n": 255, "mean": 189.82745098039214, "median": 183.0, "p90": 256.0 }, "searches": { "n": 255, "mean": 1.3176470588235294, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 255, "mean": 1.3019607843137255, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 255, "mean": 1.2470588235294118, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 255, "mean": 1.2117647058823529, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 255, "mean": 0.03529411764705882, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 255, "mean": 0.3215686274509804, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 255, "mean": 0.20392156862745098, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 255, "mean": 1.5960784313725491, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 255, "mean": 0.17647058823529413, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 255, "mean": 0.12156862745098039, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 255, "mean": 0.09803921568627451, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 255, "mean": 294.73106531275675, "median": 286.75, "p90": 399.42058823529413 }, "first_product_rank": { "n": 249, "mean": 4.493975903614458, "median": 1.0, "p90": 11.200000000000017 } }, "success_rate": 0.6039215686274509, "reward_mean": 0.6322012138188609, "pooled_tokens_per_turn": 236.19322709163347, "token_complete_episodes": 255, "rates": { "protocol_errors": 0.07450980392156863, "product_revisits": 0.03137254901960784, "option_changes": 0.06666666666666667, "backtracks": 0.1568627450980392 }, "terminations": { "purchase": 237, "generation_length": 18 } }, "success": { "n": 154, "tasks": 26, "metrics": { "model_steps": { "n": 154, "mean": 4.675324675324675, "median": 4.0, "p90": 5.0 }, "actions": { "n": 154, "mean": 4.642857142857143, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 154, "mean": 4.642857142857143, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 154, "mean": 223.7200989486704, "median": 215.71428571428572, "p90": 296.02500000000003 }, "generated_tokens": { "n": 154, "mean": 1041.6753246753246, "median": 963.5, "p90": 1412.7 }, "first_turn_tokens": { "n": 154, "mean": 190.1103896103896, "median": 184.0, "p90": 258.80000000000007 }, "searches": { "n": 154, "mean": 1.0714285714285714, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 154, "mean": 1.0649350649350648, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 154, "mean": 1.0714285714285714, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 154, "mean": 1.0584415584415585, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 154, "mean": 0.012987012987012988, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 154, "mean": 0.07142857142857142, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 154, "mean": 0.01948051948051948, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 154, "mean": 1.4090909090909092, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 154, "mean": 0.05844155844155844, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 154, "mean": 0.032467532467532464, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 154, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 154, "mean": 282.9463126159555, "median": 265.375, "p90": 389.94000000000005 }, "first_product_rank": { "n": 154, "mean": 2.792207792207792, "median": 1.0, "p90": 5.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 222.80277777777778, "token_complete_episodes": 154, "rates": { "protocol_errors": 0.01948051948051948, "product_revisits": 0.012987012987012988, "option_changes": 0.05194805194805195, "backtracks": 0.05194805194805195 }, "terminations": { "purchase": 154 } }, "failure": { "n": 101, "tasks": 23, "metrics": { "model_steps": { "n": 101, "mean": 7.782178217821782, "median": 5.0, "p90": 17.0 }, "actions": { "n": 101, "mean": 7.346534653465347, "median": 5.0, "p90": 16.0 }, "valid_actions": { "n": 101, "mean": 7.099009900990099, "median": 5.0, "p90": 15.0 }, "tokens_per_turn": { "n": 101, "mean": 265.37553242316403, "median": 239.0, "p90": 335.75 }, "generated_tokens": { "n": 101, "mean": 1933.5544554455446, "median": 1226.0, "p90": 4244.0 }, "first_turn_tokens": { "n": 101, "mean": 189.3960396039604, "median": 183.0, "p90": 251.0 }, "searches": { "n": 101, "mean": 1.693069306930693, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 101, "mean": 1.6633663366336633, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 101, "mean": 1.5148514851485149, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 101, "mean": 1.4455445544554455, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 101, "mean": 0.06930693069306931, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 101, "mean": 0.7029702970297029, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 101, "mean": 0.48514851485148514, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 101, "mean": 1.881188118811881, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 101, "mean": 0.3564356435643564, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 101, "mean": 0.25742574257425743, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 101, "mean": 0.24752475247524752, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 101, "mean": 312.6998961573845, "median": 300.6, "p90": 413.44444444444446 }, "first_product_rank": { "n": 95, "mean": 7.252631578947368, "median": 2.0, "p90": 15.600000000000009 } }, "success_rate": 0.0, "reward_mean": 0.0713991041961339, "pooled_tokens_per_turn": 248.4592875318066, "token_complete_episodes": 101, "rates": { "protocol_errors": 0.15841584158415842, "product_revisits": 0.0594059405940594, "option_changes": 0.0891089108910891, "backtracks": 0.31683168316831684 }, "terminations": { "purchase": 83, "generation_length": 18 } } }, { "step": 12, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.1875, "median": 4.0, "p90": 9.0 }, "actions": { "n": 256, "mean": 5.125, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.0625, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 260.69195947310413, "median": 233.25, "p90": 354.25 }, "generated_tokens": { "n": 256, "mean": 1326.484375, "median": 1074.5, "p90": 2243.0 }, "first_turn_tokens": { "n": 256, "mean": 187.0234375, "median": 175.0, "p90": 260.0 }, "searches": { "n": 256, "mean": 1.171875, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.1640625, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.14453125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.125, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.171875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.14453125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.4453125, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.1171875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.0625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 321.16732710220833, "median": 300.625, "p90": 447.875 }, "first_product_rank": { "n": 252, "mean": 4.507936507936508, "median": 1.0, "p90": 10.0 } }, "success_rate": 0.66796875, "reward_mean": 0.68505859375, "pooled_tokens_per_turn": 255.7078313253012, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0390625, "product_revisits": 0.01953125, "option_changes": 0.0625, "backtracks": 0.125 }, "terminations": { "purchase": 252, "generation_length": 4 } }, "success": { "n": 171, "tasks": 27, "metrics": { "model_steps": { "n": 171, "mean": 4.730994152046784, "median": 4.0, "p90": 5.0 }, "actions": { "n": 171, "mean": 4.701754385964913, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 171, "mean": 4.678362573099415, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 171, "mean": 235.46940161852442, "median": 217.25, "p90": 319.25 }, "generated_tokens": { "n": 171, "mean": 1095.7836257309941, "median": 945.0, "p90": 1720.0 }, "first_turn_tokens": { "n": 171, "mean": 184.83625730994152, "median": 175.0, "p90": 257.0 }, "searches": { "n": 171, "mean": 1.0994152046783625, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 171, "mean": 1.0994152046783625, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 171, "mean": 1.0935672514619883, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 171, "mean": 1.0760233918128654, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 171, "mean": 0.017543859649122806, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 171, "mean": 0.09941520467836257, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 171, "mean": 0.017543859649122806, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 171, "mean": 1.368421052631579, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 171, "mean": 0.14619883040935672, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 171, "mean": 0.029239766081871343, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 171, "mean": 0.023391812865497075, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 171, "mean": 297.8053104140823, "median": 281.5, "p90": 404.0 }, "first_product_rank": { "n": 171, "mean": 2.7251461988304095, "median": 1.0, "p90": 7.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 231.61804697156984, "token_complete_episodes": 171, "rates": { "protocol_errors": 0.017543859649122806, "product_revisits": 0.017543859649122806, "option_changes": 0.08187134502923976, "backtracks": 0.07602339181286549 }, "terminations": { "purchase": 171 } }, "failure": { "n": 85, "tasks": 16, "metrics": { "model_steps": { "n": 85, "mean": 6.105882352941176, "median": 5.0, "p90": 11.0 }, "actions": { "n": 85, "mean": 5.976470588235294, "median": 5.0, "p90": 10.0 }, "valid_actions": { "n": 85, "mean": 5.8352941176470585, "median": 5.0, "p90": 10.0 }, "tokens_per_turn": { "n": 85, "mean": 311.43381115702323, "median": 283.2, "p90": 398.6600000000001 }, "generated_tokens": { "n": 85, "mean": 1790.6, "median": 1416.0, "p90": 3517.8000000000025 }, "first_turn_tokens": { "n": 85, "mean": 191.42352941176472, "median": 181.0, "p90": 261.8 }, "searches": { "n": 85, "mean": 1.3176470588235294, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 85, "mean": 1.2941176470588236, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 85, "mean": 1.2470588235294118, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 85, "mean": 1.223529411764706, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 85, "mean": 0.023529411764705882, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 85, "mean": 0.3176470588235294, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 85, "mean": 0.4, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 85, "mean": 1.6, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 85, "mean": 0.058823529411764705, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 85, "mean": 0.08235294117647059, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 85, "mean": 0.1411764705882353, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 85, "mean": 368.1662077336149, "median": 358.4, "p90": 508.1000000000001 }, "first_product_rank": { "n": 81, "mean": 8.271604938271604, "median": 4.0, "p90": 16.0 } }, "success_rate": 0.0, "reward_mean": 0.051470588235294115, "pooled_tokens_per_turn": 293.2581888246628, "token_complete_episodes": 85, "rates": { "protocol_errors": 0.08235294117647059, "product_revisits": 0.023529411764705882, "option_changes": 0.023529411764705882, "backtracks": 0.2235294117647059 }, "terminations": { "purchase": 81, "generation_length": 4 } } }, { "step": 13, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.54296875, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.5078125, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.49609375, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 227.45096112112714, "median": 216.6153846153846, "p90": 310.0 }, "generated_tokens": { "n": 256, "mean": 1030.3828125, "median": 943.0, "p90": 1465.0 }, "first_turn_tokens": { "n": 256, "mean": 186.7265625, "median": 179.5, "p90": 261.5 }, "searches": { "n": 256, "mean": 1.05078125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.05078125, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.046875, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0390625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.328125, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 287.2758776542468, "median": 276.25, "p90": 397.375 }, "first_product_rank": { "n": 256, "mean": 3.328125, "median": 1.0, "p90": 10.0 } }, "success_rate": 0.734375, "reward_mean": 0.75, "pooled_tokens_per_turn": 226.80825451418744, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.03515625, "product_revisits": 0.0078125, "option_changes": 0.015625, "backtracks": 0.04296875 }, "terminations": { "purchase": 256 } }, "success": { "n": 188, "tasks": 27, "metrics": { "model_steps": { "n": 188, "mean": 4.553191489361702, "median": 4.0, "p90": 5.0 }, "actions": { "n": 188, "mean": 4.526595744680851, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 188, "mean": 4.5212765957446805, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 188, "mean": 219.90693819330787, "median": 211.75, "p90": 296.5400000000001 }, "generated_tokens": { "n": 188, "mean": 997.531914893617, "median": 884.5, "p90": 1351.6000000000001 }, "first_turn_tokens": { "n": 188, "mean": 184.40425531914894, "median": 173.5, "p90": 258.6 }, "searches": { "n": 188, "mean": 1.047872340425532, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 188, "mean": 1.047872340425532, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 188, "mean": 1.0425531914893618, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 188, "mean": 1.0319148936170213, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 188, "mean": 0.010638297872340425, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 188, "mean": 0.047872340425531915, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 188, "mean": 0.02127659574468085, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 188, "mean": 1.3617021276595744, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 188, "mean": 0.026595744680851064, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 188, "mean": 0.026595744680851064, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 188, "mean": 0.005319148936170213, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 188, "mean": 275.6262394298964, "median": 267.0, "p90": 384.325 }, "first_product_rank": { "n": 188, "mean": 3.0372340425531914, "median": 1.0, "p90": 9.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 219.0841121495327, "token_complete_episodes": 188, "rates": { "protocol_errors": 0.026595744680851064, "product_revisits": 0.010638297872340425, "option_changes": 0.010638297872340425, "backtracks": 0.03723404255319149 }, "terminations": { "purchase": 188 } }, "failure": { "n": 68, "tasks": 15, "metrics": { "model_steps": { "n": 68, "mean": 4.514705882352941, "median": 4.0, "p90": 5.0 }, "actions": { "n": 68, "mean": 4.455882352941177, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 68, "mean": 4.426470588235294, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 68, "mean": 248.30796568627449, "median": 235.625, "p90": 351.825 }, "generated_tokens": { "n": 68, "mean": 1121.2058823529412, "median": 994.0, "p90": 1510.0 }, "first_turn_tokens": { "n": 68, "mean": 193.14705882352942, "median": 193.5, "p90": 271.20000000000005 }, "searches": { "n": 68, "mean": 1.0588235294117647, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 68, "mean": 1.0588235294117647, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 68, "mean": 1.0588235294117647, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 68, "mean": 1.0588235294117647, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 68, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 68, "mean": 0.058823529411764705, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 68, "mean": 0.014705882352941176, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 68, "mean": 1.2352941176470589, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 68, "mean": 0.029411764705882353, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 68, "mean": 0.058823529411764705, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 68, "mean": 0.029411764705882353, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 68, "mean": 319.4837009803922, "median": 312.0, "p90": 430.1500000000001 }, "first_product_rank": { "n": 68, "mean": 4.132352941176471, "median": 1.0, "p90": 13.0 } }, "success_rate": 0.0, "reward_mean": 0.058823529411764705, "pooled_tokens_per_turn": 248.34527687296418, "token_complete_episodes": 68, "rates": { "protocol_errors": 0.058823529411764705, "product_revisits": 0.0, "option_changes": 0.029411764705882353, "backtracks": 0.058823529411764705 }, "terminations": { "purchase": 68 } } }, { "step": 14, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.8359375, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.77734375, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.70703125, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 238.206753551182, "median": 218.675, "p90": 316.25 }, "generated_tokens": { "n": 256, "mean": 1157.546875, "median": 916.5, "p90": 1672.0 }, "first_turn_tokens": { "n": 256, "mean": 168.2734375, "median": 154.5, "p90": 238.5 }, "searches": { "n": 256, "mean": 1.13671875, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.13671875, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.12109375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.109375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.13671875, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.30078125, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0546875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.0703125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 293.25571697582393, "median": 277.75, "p90": 398.375 }, "first_product_rank": { "n": 255, "mean": 3.4235294117647057, "median": 1.0, "p90": 9.599999999999994 } }, "success_rate": 0.6171875, "reward_mean": 0.6458333333333334, "pooled_tokens_per_turn": 239.36348949919224, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.03125, "product_revisits": 0.0078125, "option_changes": 0.01953125, "backtracks": 0.0859375 }, "terminations": { "purchase": 252, "generation_length": 4 } }, "success": { "n": 158, "tasks": 27, "metrics": { "model_steps": { "n": 158, "mean": 4.2594936708860756, "median": 4.0, "p90": 5.0 }, "actions": { "n": 158, "mean": 4.2594936708860756, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 158, "mean": 4.2594936708860756, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 158, "mean": 215.0362341772152, "median": 212.375, "p90": 278.475 }, "generated_tokens": { "n": 158, "mean": 907.7974683544304, "median": 896.0, "p90": 1138.5 }, "first_turn_tokens": { "n": 158, "mean": 161.18354430379748, "median": 149.0, "p90": 225.4000000000001 }, "searches": { "n": 158, "mean": 1.0126582278481013, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 158, "mean": 1.0126582278481013, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 158, "mean": 1.0126582278481013, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 158, "mean": 1.0126582278481013, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 158, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 158, "mean": 0.012658227848101266, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 158, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 158, "mean": 1.2215189873417722, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 158, "mean": 0.012658227848101266, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 158, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 158, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 158, "mean": 274.2386075949367, "median": 269.15, "p90": 358.75 }, "first_product_rank": { "n": 158, "mean": 2.411392405063291, "median": 1.0, "p90": 7.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 213.12332838038634, "token_complete_episodes": 158, "rates": { "protocol_errors": 0.0, "product_revisits": 0.0, "option_changes": 0.012658227848101266, "backtracks": 0.012658227848101266 }, "terminations": { "purchase": 158 } }, "failure": { "n": 98, "tasks": 18, "metrics": { "model_steps": { "n": 98, "mean": 5.76530612244898, "median": 4.0, "p90": 9.0 }, "actions": { "n": 98, "mean": 5.612244897959184, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 98, "mean": 5.428571428571429, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 98, "mean": 275.56330519492445, "median": 228.375, "p90": 414.625 }, "generated_tokens": { "n": 98, "mean": 1560.204081632653, "median": 1086.0, "p90": 3239.4999999999995 }, "first_turn_tokens": { "n": 98, "mean": 179.70408163265307, "median": 176.5, "p90": 241.89999999999998 }, "searches": { "n": 98, "mean": 1.336734693877551, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 98, "mean": 1.336734693877551, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 98, "mean": 1.2959183673469388, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 98, "mean": 1.2653061224489797, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 98, "mean": 0.030612244897959183, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 98, "mean": 0.336734693877551, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 98, "mean": 0.07142857142857142, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 98, "mean": 1.4285714285714286, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 98, "mean": 0.12244897959183673, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 98, "mean": 0.11224489795918367, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 98, "mean": 0.1836734693877551, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 98, "mean": 323.91595454909094, "median": 289.25, "p90": 500.0874999999999 }, "first_product_rank": { "n": 97, "mean": 5.072164948453608, "median": 2.0, "p90": 12.0 } }, "success_rate": 0.0, "reward_mean": 0.07482993197278912, "pooled_tokens_per_turn": 270.6194690265487, "token_complete_episodes": 98, "rates": { "protocol_errors": 0.08163265306122448, "product_revisits": 0.02040816326530612, "option_changes": 0.030612244897959183, "backtracks": 0.20408163265306123 }, "terminations": { "purchase": 94, "generation_length": 4 } } }, { "step": 15, "all": { "n": 251, "tasks": 32, "metrics": { "model_steps": { "n": 251, "mean": 5.247011952191235, "median": 4.0, "p90": 8.0 }, "actions": { "n": 251, "mean": 5.171314741035856, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 251, "mean": 5.131474103585657, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 251, "mean": 257.56093598743, "median": 241.4, "p90": 369.7 }, "generated_tokens": { "n": 251, "mean": 1318.3386454183267, "median": 1094.0, "p90": 2218.0 }, "first_turn_tokens": { "n": 251, "mean": 193.85258964143426, "median": 182.0, "p90": 267.0 }, "searches": { "n": 251, "mean": 1.2191235059760956, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 251, "mean": 1.2191235059760956, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 251, "mean": 1.1633466135458168, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 251, "mean": 1.1474103585657371, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 251, "mean": 0.01593625498007968, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 251, "mean": 0.21912350597609562, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 251, "mean": 0.18326693227091634, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 251, "mean": 1.3665338645418326, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 251, "mean": 0.0199203187250996, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 251, "mean": 0.055776892430278883, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 251, "mean": 0.0398406374501992, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 251, "mean": 327.13087445975896, "median": 303.8, "p90": 470.5 }, "first_product_rank": { "n": 248, "mean": 5.270161290322581, "median": 1.0, "p90": 14.0 } }, "success_rate": 0.6454183266932271, "reward_mean": 0.6570385126162018, "pooled_tokens_per_turn": 251.25512528473803, "token_complete_episodes": 251, "rates": { "protocol_errors": 0.04780876494023904, "product_revisits": 0.01593625498007968, "option_changes": 0.01593625498007968, "backtracks": 0.1394422310756972 }, "terminations": { "purchase": 246, "generation_length": 5 } }, "success": { "n": 162, "tasks": 29, "metrics": { "model_steps": { "n": 162, "mean": 4.777777777777778, "median": 4.0, "p90": 5.0 }, "actions": { "n": 162, "mean": 4.734567901234568, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 162, "mean": 4.722222222222222, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 162, "mean": 242.66398195425975, "median": 218.875, "p90": 353.675 }, "generated_tokens": { "n": 162, "mean": 1132.5493827160494, "median": 1023.0, "p90": 1609.6000000000004 }, "first_turn_tokens": { "n": 162, "mean": 197.7716049382716, "median": 183.0, "p90": 281.20000000000005 }, "searches": { "n": 162, "mean": 1.1049382716049383, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 162, "mean": 1.1049382716049383, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 162, "mean": 1.0679012345679013, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 162, "mean": 1.0617283950617284, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 162, "mean": 0.006172839506172839, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 162, "mean": 0.10493827160493827, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 162, "mean": 0.06172839506172839, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 162, "mean": 1.382716049382716, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 162, "mean": 0.012345679012345678, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 162, "mean": 0.043209876543209874, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 162, "mean": 0.012345679012345678, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 162, "mean": 306.7323073531407, "median": 273.625, "p90": 438.59000000000003 }, "first_product_rank": { "n": 162, "mean": 3.7160493827160495, "median": 1.0, "p90": 11.900000000000006 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 237.04521963824288, "token_complete_episodes": 162, "rates": { "protocol_errors": 0.037037037037037035, "product_revisits": 0.006172839506172839, "option_changes": 0.012345679012345678, "backtracks": 0.07407407407407407 }, "terminations": { "purchase": 162 } }, "failure": { "n": 89, "tasks": 21, "metrics": { "model_steps": { "n": 89, "mean": 6.101123595505618, "median": 4.0, "p90": 11.0 }, "actions": { "n": 89, "mean": 5.966292134831461, "median": 4.0, "p90": 10.0 }, "valid_actions": { "n": 89, "mean": 5.876404494382022, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 89, "mean": 284.6767399579198, "median": 270.0, "p90": 395.02 }, "generated_tokens": { "n": 89, "mean": 1656.5168539325844, "median": 1293.0, "p90": 3138.0000000000005 }, "first_turn_tokens": { "n": 89, "mean": 186.7191011235955, "median": 175.0, "p90": 251.60000000000002 }, "searches": { "n": 89, "mean": 1.4269662921348314, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 89, "mean": 1.4269662921348314, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 89, "mean": 1.3370786516853932, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 89, "mean": 1.303370786516854, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 89, "mean": 0.033707865168539325, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 89, "mean": 0.42696629213483145, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 89, "mean": 0.4044943820224719, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 89, "mean": 1.3370786516853932, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 89, "mean": 0.033707865168539325, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 89, "mean": 0.07865168539325842, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 89, "mean": 0.0898876404494382, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 89, "mean": 364.2608505414685, "median": 341.25, "p90": 501.5 }, "first_product_rank": { "n": 86, "mean": 8.19767441860465, "median": 4.0, "p90": 19.0 } }, "success_rate": 0.0, "reward_mean": 0.03277153558052434, "pooled_tokens_per_turn": 271.51012891344385, "token_complete_episodes": 89, "rates": { "protocol_errors": 0.06741573033707865, "product_revisits": 0.033707865168539325, "option_changes": 0.02247191011235955, "backtracks": 0.25842696629213485 }, "terminations": { "purchase": 84, "generation_length": 5 } } }, { "step": 16, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.5546875, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.53515625, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.48828125, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 250.3842325697306, "median": 237.375, "p90": 323.25 }, "generated_tokens": { "n": 256, "mean": 1138.8515625, "median": 1005.5, "p90": 1664.0 }, "first_turn_tokens": { "n": 256, "mean": 182.62890625, "median": 175.0, "p90": 256.5 }, "searches": { "n": 256, "mean": 1.10546875, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.1015625, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.09375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.08203125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.10546875, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.16015625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 316.72251882774344, "median": 298.75, "p90": 426.625 }, "first_product_rank": { "n": 255, "mean": 2.7058823529411766, "median": 1.0, "p90": 8.0 } }, "success_rate": 0.7890625, "reward_mean": 0.7890625, "pooled_tokens_per_turn": 250.03945111492283, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.01171875, "product_revisits": 0.01171875, "option_changes": 0.0078125, "backtracks": 0.078125 }, "terminations": { "purchase": 255, "generation_length": 1 } }, "success": { "n": 202, "tasks": 31, "metrics": { "model_steps": { "n": 202, "mean": 4.292079207920792, "median": 4.0, "p90": 5.0 }, "actions": { "n": 202, "mean": 4.292079207920792, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 202, "mean": 4.287128712871287, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 202, "mean": 245.5514497878359, "median": 233.75, "p90": 319.975 }, "generated_tokens": { "n": 202, "mean": 1056.2029702970297, "median": 961.0, "p90": 1500.2000000000003 }, "first_turn_tokens": { "n": 202, "mean": 180.64851485148515, "median": 171.5, "p90": 247.0 }, "searches": { "n": 202, "mean": 1.0495049504950495, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 202, "mean": 1.0495049504950495, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 202, "mean": 1.0495049504950495, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 202, "mean": 1.0445544554455446, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 202, "mean": 0.0049504950495049506, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 202, "mean": 0.04950495049504951, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 202, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 202, "mean": 1.1386138613861385, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 202, "mean": 0.0049504950495049506, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 202, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 202, "mean": 0.0049504950495049506, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 202, "mean": 307.922654408298, "median": 290.125, "p90": 412.76500000000004 }, "first_product_rank": { "n": 202, "mean": 2.227722772277228, "median": 1.0, "p90": 4.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 246.08189158016148, "token_complete_episodes": 202, "rates": { "protocol_errors": 0.0, "product_revisits": 0.0049504950495049506, "option_changes": 0.0049504950495049506, "backtracks": 0.034653465346534656 }, "terminations": { "purchase": 202 } }, "failure": { "n": 54, "tasks": 13, "metrics": { "model_steps": { "n": 54, "mean": 5.537037037037037, "median": 4.0, "p90": 10.0 }, "actions": { "n": 54, "mean": 5.444444444444445, "median": 4.0, "p90": 9.0 }, "valid_actions": { "n": 54, "mean": 5.2407407407407405, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 54, "mean": 268.46242001311447, "median": 258.875, "p90": 336.27500000000003 }, "generated_tokens": { "n": 54, "mean": 1448.0185185185185, "median": 1199.5, "p90": 2608.9000000000005 }, "first_turn_tokens": { "n": 54, "mean": 190.03703703703704, "median": 193.5, "p90": 261.7 }, "searches": { "n": 54, "mean": 1.3148148148148149, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 54, "mean": 1.2962962962962963, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 54, "mean": 1.2592592592592593, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 54, "mean": 1.2222222222222223, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 54, "mean": 0.037037037037037035, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 54, "mean": 0.3148148148148148, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 54, "mean": 0.12962962962962962, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 54, "mean": 1.2407407407407407, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 54, "mean": 0.018518518518518517, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 54, "mean": 0.07407407407407407, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 54, "mean": 0.2037037037037037, "median": 0.0, "p90": 0.7000000000000028 }, "reasoning_chars_per_turn": { "n": 54, "mean": 349.64053017455797, "median": 337.67857142857144, "p90": 433.225 }, "first_product_rank": { "n": 53, "mean": 4.528301886792453, "median": 3.0, "p90": 10.600000000000009 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 261.5150501672241, "token_complete_episodes": 54, "rates": { "protocol_errors": 0.05555555555555555, "product_revisits": 0.037037037037037035, "option_changes": 0.018518518518518517, "backtracks": 0.24074074074074073 }, "terminations": { "generation_length": 1, "purchase": 53 } } }, { "step": 17, "all": { "n": 255, "tasks": 32, "metrics": { "model_steps": { "n": 255, "mean": 4.466666666666667, "median": 4.0, "p90": 5.0 }, "actions": { "n": 255, "mean": 4.447058823529412, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 255, "mean": 4.392156862745098, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 255, "mean": 275.7212962962963, "median": 264.4, "p90": 373.4 }, "generated_tokens": { "n": 255, "mean": 1223.2509803921569, "median": 1133.0, "p90": 1641.8 }, "first_turn_tokens": { "n": 255, "mean": 196.15686274509804, "median": 193.0, "p90": 265.6 }, "searches": { "n": 255, "mean": 1.0549019607843138, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 255, "mean": 1.0549019607843138, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 255, "mean": 1.0549019607843138, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 255, "mean": 1.0509803921568628, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 255, "mean": 0.00392156862745098, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 255, "mean": 0.054901960784313725, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 255, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 255, "mean": 1.227450980392157, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 255, "mean": 0.0196078431372549, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 255, "mean": 0.0196078431372549, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 255, "mean": 0.054901960784313725, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 255, "mean": 350.3699066293184, "median": 340.75, "p90": 477.0 }, "first_product_rank": { "n": 255, "mean": 2.627450980392157, "median": 1.0, "p90": 5.599999999999994 } }, "success_rate": 0.7725490196078432, "reward_mean": 0.7774509803921569, "pooled_tokens_per_turn": 273.86215978928885, "token_complete_episodes": 255, "rates": { "protocol_errors": 0.011764705882352941, "product_revisits": 0.00392156862745098, "option_changes": 0.011764705882352941, "backtracks": 0.043137254901960784 }, "terminations": { "purchase": 255 } }, "success": { "n": 197, "tasks": 30, "metrics": { "model_steps": { "n": 197, "mean": 4.416243654822335, "median": 4.0, "p90": 5.0 }, "actions": { "n": 197, "mean": 4.401015228426396, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 197, "mean": 4.365482233502538, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 197, "mean": 265.1347997743937, "median": 249.0, "p90": 362.2 }, "generated_tokens": { "n": 197, "mean": 1162.791878172589, "median": 1079.0, "p90": 1524.0 }, "first_turn_tokens": { "n": 197, "mean": 193.80710659898477, "median": 188.0, "p90": 257.6 }, "searches": { "n": 197, "mean": 1.0406091370558375, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 197, "mean": 1.0406091370558375, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 197, "mean": 1.0406091370558375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 197, "mean": 1.0355329949238579, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 197, "mean": 0.005076142131979695, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 197, "mean": 0.04060913705583756, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 197, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 197, "mean": 1.2436548223350254, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 197, "mean": 0.015228426395939087, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 197, "mean": 0.015228426395939087, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 197, "mean": 0.03553299492385787, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 197, "mean": 336.4982656514382, "median": 321.5, "p90": 463.20000000000005 }, "first_product_rank": { "n": 197, "mean": 2.3401015228426396, "median": 1.0, "p90": 5.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 263.2988505747126, "token_complete_episodes": 197, "rates": { "protocol_errors": 0.01015228426395939, "product_revisits": 0.005076142131979695, "option_changes": 0.005076142131979695, "backtracks": 0.03553299492385787 }, "terminations": { "purchase": 197 } }, "failure": { "n": 58, "tasks": 18, "metrics": { "model_steps": { "n": 58, "mean": 4.637931034482759, "median": 4.0, "p90": 5.0 }, "actions": { "n": 58, "mean": 4.603448275862069, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 58, "mean": 4.482758620689655, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 58, "mean": 311.6788793103448, "median": 304.775, "p90": 410.5825 }, "generated_tokens": { "n": 58, "mean": 1428.603448275862, "median": 1233.5, "p90": 2017.3000000000004 }, "first_turn_tokens": { "n": 58, "mean": 204.13793103448276, "median": 206.0, "p90": 278.6 }, "searches": { "n": 58, "mean": 1.103448275862069, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 58, "mean": 1.103448275862069, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 58, "mean": 1.103448275862069, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 58, "mean": 1.103448275862069, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 58, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 58, "mean": 0.10344827586206896, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 58, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 58, "mean": 1.1724137931034482, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 58, "mean": 0.034482758620689655, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 58, "mean": 0.034482758620689655, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 58, "mean": 0.1206896551724138, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 58, "mean": 397.4856527093596, "median": 385.375, "p90": 541.8625000000001 }, "first_product_rank": { "n": 58, "mean": 3.603448275862069, "median": 2.0, "p90": 8.300000000000004 } }, "success_rate": 0.0, "reward_mean": 0.021551724137931036, "pooled_tokens_per_turn": 308.02602230483274, "token_complete_episodes": 58, "rates": { "protocol_errors": 0.017241379310344827, "product_revisits": 0.0, "option_changes": 0.034482758620689655, "backtracks": 0.06896551724137931 }, "terminations": { "purchase": 58 } } }, { "step": 18, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.6484375, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.640625, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.63671875, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 260.03484823224295, "median": 248.85, "p90": 344.25 }, "generated_tokens": { "n": 256, "mean": 1210.05859375, "median": 1092.0, "p90": 1756.0 }, "first_turn_tokens": { "n": 256, "mean": 196.671875, "median": 188.0, "p90": 272.5 }, "searches": { "n": 256, "mean": 1.1015625, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.1015625, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.1015625, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0859375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.1015625, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.3203125, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 335.0538416218137, "median": 323.9, "p90": 455.75 }, "first_product_rank": { "n": 256, "mean": 3.99609375, "median": 1.0, "p90": 11.5 } }, "success_rate": 0.57421875, "reward_mean": 0.5879603794642857, "pooled_tokens_per_turn": 260.3151260504202, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0078125, "product_revisits": 0.015625, "option_changes": 0.00390625, "backtracks": 0.05859375 }, "terminations": { "purchase": 256 } }, "success": { "n": 147, "tasks": 21, "metrics": { "model_steps": { "n": 147, "mean": 4.333333333333333, "median": 4.0, "p90": 5.0 }, "actions": { "n": 147, "mean": 4.333333333333333, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 147, "mean": 4.333333333333333, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 147, "mean": 247.7118858654573, "median": 237.5, "p90": 313.85 }, "generated_tokens": { "n": 147, "mean": 1071.3265306122448, "median": 1006.0, "p90": 1411.0000000000005 }, "first_turn_tokens": { "n": 147, "mean": 185.18367346938774, "median": 177.0, "p90": 243.4 }, "searches": { "n": 147, "mean": 1.0204081632653061, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 147, "mean": 1.0204081632653061, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 147, "mean": 1.0204081632653061, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 147, "mean": 1.0204081632653061, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 147, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 147, "mean": 0.02040816326530612, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 147, "mean": 0.013605442176870748, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 147, "mean": 1.2585034013605443, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 147, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 147, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 147, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 147, "mean": 315.83293650793655, "median": 306.25, "p90": 395.8 }, "first_product_rank": { "n": 147, "mean": 3.6530612244897958, "median": 2.0, "p90": 9.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 247.2291993720565, "token_complete_episodes": 147, "rates": { "protocol_errors": 0.0, "product_revisits": 0.0, "option_changes": 0.0, "backtracks": 0.02040816326530612 }, "terminations": { "purchase": 147 } }, "failure": { "n": 109, "tasks": 19, "metrics": { "model_steps": { "n": 109, "mean": 5.073394495412844, "median": 4.0, "p90": 7.200000000000003 }, "actions": { "n": 109, "mean": 5.055045871559633, "median": 4.0, "p90": 7.200000000000003 }, "valid_actions": { "n": 109, "mean": 5.045871559633028, "median": 4.0, "p90": 7.200000000000003 }, "tokens_per_turn": { "n": 109, "mean": 276.6538892223117, "median": 268.27272727272725, "p90": 368.7771929824562 }, "generated_tokens": { "n": 109, "mean": 1397.1559633027523, "median": 1177.0, "p90": 2015.0 }, "first_turn_tokens": { "n": 109, "mean": 212.1651376146789, "median": 205.0, "p90": 281.8 }, "searches": { "n": 109, "mean": 1.2110091743119267, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 109, "mean": 1.2110091743119267, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 109, "mean": 1.2110091743119267, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 109, "mean": 1.1743119266055047, "median": 1.0, "p90": 1.2000000000000028 }, "product_revisits": { "n": 109, "mean": 0.03669724770642202, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 109, "mean": 0.21100917431192662, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 109, "mean": 0.009174311926605505, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 109, "mean": 1.4036697247706422, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 109, "mean": 0.009174311926605505, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 109, "mean": 0.01834862385321101, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 109, "mean": 0.009174311926605505, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 109, "mean": 360.9756127386939, "median": 348.75, "p90": 490.8 }, "first_product_rank": { "n": 109, "mean": 4.458715596330276, "median": 1.0, "p90": 14.200000000000003 } }, "success_rate": 0.0, "reward_mean": 0.03227391874180865, "pooled_tokens_per_turn": 275.3887884267631, "token_complete_episodes": 109, "rates": { "protocol_errors": 0.01834862385321101, "product_revisits": 0.03669724770642202, "option_changes": 0.009174311926605505, "backtracks": 0.11009174311926606 }, "terminations": { "purchase": 109 } } }, { "step": 19, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.72265625, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.484375, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.48046875, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 254.97199942527095, "median": 246.5, "p90": 333.20833333333337 }, "generated_tokens": { "n": 256, "mean": 1247.35546875, "median": 1048.5, "p90": 1690.0 }, "first_turn_tokens": { "n": 256, "mean": 184.85546875, "median": 178.0, "p90": 252.0 }, "searches": { "n": 256, "mean": 1.08203125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.08203125, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.0703125, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.06640625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.08203125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.24609375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.234375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 325.29601731818525, "median": 312.35, "p90": 439.5625 }, "first_product_rank": { "n": 254, "mean": 2.6496062992125986, "median": 1.0, "p90": 7.0 } }, "success_rate": 0.8046875, "reward_mean": 0.8119436553030303, "pooled_tokens_per_turn": 264.12158808933003, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.05078125, "product_revisits": 0.00390625, "option_changes": 0.0, "backtracks": 0.0625 }, "terminations": { "purchase": 254, "generation_length": 1, "action_limit": 1 } }, "success": { "n": 206, "tasks": 29, "metrics": { "model_steps": { "n": 206, "mean": 4.330097087378641, "median": 4.0, "p90": 5.0 }, "actions": { "n": 206, "mean": 4.300970873786408, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 206, "mean": 4.296116504854369, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 206, "mean": 250.72719795037756, "median": 242.5, "p90": 325.5625 }, "generated_tokens": { "n": 206, "mean": 1088.8300970873786, "median": 1004.5, "p90": 1420.0 }, "first_turn_tokens": { "n": 206, "mean": 187.1747572815534, "median": 180.5, "p90": 257.0 }, "searches": { "n": 206, "mean": 1.029126213592233, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 206, "mean": 1.029126213592233, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 206, "mean": 1.0242718446601942, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 206, "mean": 1.0194174757281553, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 206, "mean": 0.0048543689320388345, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 206, "mean": 0.02912621359223301, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 206, "mean": 0.009708737864077669, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 206, "mean": 1.203883495145631, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 206, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 206, "mean": 0.02912621359223301, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 206, "mean": 0.0048543689320388345, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 206, "mean": 318.24848975188775, "median": 307.625, "p90": 425.75 }, "first_product_rank": { "n": 206, "mean": 2.3058252427184467, "median": 1.0, "p90": 5.5 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 251.45627802690584, "token_complete_episodes": 206, "rates": { "protocol_errors": 0.024271844660194174, "product_revisits": 0.0048543689320388345, "option_changes": 0.0, "backtracks": 0.02912621359223301 }, "terminations": { "purchase": 206 } }, "failure": { "n": 50, "tasks": 13, "metrics": { "model_steps": { "n": 50, "mean": 6.34, "median": 4.0, "p90": 10.300000000000004 }, "actions": { "n": 50, "mean": 5.24, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 50, "mean": 5.24, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 50, "mean": 272.4605815018315, "median": 267.375, "p90": 370.32142857142856 }, "generated_tokens": { "n": 50, "mean": 1900.48, "median": 1178.5, "p90": 3061.000000000001 }, "first_turn_tokens": { "n": 50, "mean": 175.3, "median": 157.5, "p90": 237.4 }, "searches": { "n": 50, "mean": 1.3, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 50, "mean": 1.3, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 50, "mean": 1.26, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 50, "mean": 1.26, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 50, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 50, "mean": 0.3, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 50, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 50, "mean": 1.42, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 50, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 50, "mean": 1.08, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 50, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 50, "mean": 354.3318308913309, "median": 340.0, "p90": 485.81 }, "first_product_rank": { "n": 48, "mean": 4.125, "median": 2.0, "p90": 10.300000000000004 } }, "success_rate": 0.0, "reward_mean": 0.03715151515151515, "pooled_tokens_per_turn": 299.7602523659306, "token_complete_episodes": 50, "rates": { "protocol_errors": 0.16, "product_revisits": 0.0, "option_changes": 0.0, "backtracks": 0.2 }, "terminations": { "purchase": 48, "generation_length": 1, "action_limit": 1 } } }, { "step": 20, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.1015625, "median": 4.0, "p90": 12.5 }, "actions": { "n": 256, "mean": 5.1953125, "median": 4.0, "p90": 9.0 }, "valid_actions": { "n": 256, "mean": 5.15234375, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 256, "mean": 263.40764463840816, "median": 247.125, "p90": 361.225 }, "generated_tokens": { "n": 256, "mean": 1644.73046875, "median": 1038.5, "p90": 3379.5 }, "first_turn_tokens": { "n": 256, "mean": 197.3984375, "median": 188.0, "p90": 266.5 }, "searches": { "n": 256, "mean": 1.26953125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.2578125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.21875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.1328125, "median": 1.0, "p90": 1.5 }, "product_revisits": { "n": 256, "mean": 0.0859375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.2734375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.4375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.1328125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.859375, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 338.9397307072644, "median": 311.58333333333337, "p90": 477.79999999999995 }, "first_product_rank": { "n": 250, "mean": 2.268, "median": 1.0, "p90": 4.0 } }, "success_rate": 0.71875, "reward_mean": 0.72607421875, "pooled_tokens_per_turn": 269.55889884763127, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.140625, "product_revisits": 0.0703125, "option_changes": 0.0625, "backtracks": 0.1328125 }, "terminations": { "purchase": 238, "action_limit": 6, "generation_length": 12 } }, "success": { "n": 184, "tasks": 28, "metrics": { "model_steps": { "n": 184, "mean": 4.440217391304348, "median": 4.0, "p90": 5.0 }, "actions": { "n": 184, "mean": 4.331521739130435, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 184, "mean": 4.331521739130435, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 184, "mean": 240.06714975845406, "median": 227.875, "p90": 297.1 }, "generated_tokens": { "n": 184, "mean": 1077.608695652174, "median": 938.5, "p90": 1429.1000000000008 }, "first_turn_tokens": { "n": 184, "mean": 187.95108695652175, "median": 182.0, "p90": 256.70000000000005 }, "searches": { "n": 184, "mean": 1.059782608695652, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 184, "mean": 1.059782608695652, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 184, "mean": 1.059782608695652, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 184, "mean": 1.0434782608695652, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 184, "mean": 0.016304347826086956, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 184, "mean": 0.059782608695652176, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 184, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 184, "mean": 1.1521739130434783, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 184, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 184, "mean": 0.10869565217391304, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 184, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 184, "mean": 304.7464975845411, "median": 288.125, "p90": 386.35 }, "first_product_rank": { "n": 184, "mean": 1.7445652173913044, "median": 1.0, "p90": 3.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 242.69277845777233, "token_complete_episodes": 184, "rates": { "protocol_errors": 0.09239130434782608, "product_revisits": 0.010869565217391304, "option_changes": 0.0, "backtracks": 0.03804347826086957 }, "terminations": { "purchase": 184 } }, "failure": { "n": 72, "tasks": 18, "metrics": { "model_steps": { "n": 72, "mean": 10.347222222222221, "median": 6.0, "p90": 19.9 }, "actions": { "n": 72, "mean": 7.402777777777778, "median": 5.0, "p90": 16.0 }, "valid_actions": { "n": 72, "mean": 7.25, "median": 5.0, "p90": 15.899999999999999 }, "tokens_per_turn": { "n": 72, "mean": 323.0555759982908, "median": 320.5333333333333, "p90": 456.91666666666663 }, "generated_tokens": { "n": 72, "mean": 3094.0416666666665, "median": 2098.0, "p90": 5932.2 }, "first_turn_tokens": { "n": 72, "mean": 221.54166666666666, "median": 218.0, "p90": 313.4 }, "searches": { "n": 72, "mean": 1.8055555555555556, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 72, "mean": 1.7638888888888888, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 72, "mean": 1.625, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 72, "mean": 1.3611111111111112, "median": 1.0, "p90": 2.8999999999999986 }, "product_revisits": { "n": 72, "mean": 0.2638888888888889, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 72, "mean": 0.8194444444444444, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 72, "mean": 0.08333333333333333, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 72, "mean": 2.1666666666666665, "median": 2.0, "p90": 4.0 }, "option_changes": { "n": 72, "mean": 0.4722222222222222, "median": 0.0, "p90": 1.0 }, "protocol_errors": { "n": 72, "mean": 2.7777777777777777, "median": 0.0, "p90": 4.799999999999997 }, "invalid_actions": { "n": 72, "mean": 0.1527777777777778, "median": 0.0, "p90": 0.8999999999999986 }, "reasoning_chars_per_turn": { "n": 72, "mean": 426.32243757644625, "median": 405.15, "p90": 617.22 }, "first_product_rank": { "n": 66, "mean": 3.727272727272727, "median": 1.0, "p90": 9.0 } }, "success_rate": 0.0, "reward_mean": 0.026041666666666668, "pooled_tokens_per_turn": 299.0214765100671, "token_complete_episodes": 72, "rates": { "protocol_errors": 0.2638888888888889, "product_revisits": 0.2222222222222222, "option_changes": 0.2222222222222222, "backtracks": 0.375 }, "terminations": { "purchase": 54, "action_limit": 6, "generation_length": 12 } } }, { "step": 21, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.234375, "median": 4.0, "p90": 10.0 }, "actions": { "n": 256, "mean": 5.48046875, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.06640625, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 268.6662179041839, "median": 258.15, "p90": 348.875 }, "generated_tokens": { "n": 256, "mean": 1600.55859375, "median": 1181.5, "p90": 2933.5 }, "first_turn_tokens": { "n": 256, "mean": 223.32421875, "median": 221.5, "p90": 283.5 }, "searches": { "n": 256, "mean": 1.2421875, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.23046875, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.1875, "median": 1.0, "p90": 1.5 }, "unique_products": { "n": 256, "mean": 1.10546875, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.08203125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.2421875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.10546875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.35546875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.71484375, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.4140625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 346.57559189991, "median": 335.625, "p90": 459.75 }, "first_product_rank": { "n": 249, "mean": 3.0080321285140563, "median": 1.0, "p90": 9.0 } }, "success_rate": 0.85546875, "reward_mean": 0.8606770833333333, "pooled_tokens_per_turn": 256.7312030075188, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.1484375, "product_revisits": 0.05078125, "option_changes": 0.03125, "backtracks": 0.11328125 }, "terminations": { "purchase": 239, "action_limit": 7, "generation_length": 10 } }, "success": { "n": 219, "tasks": 31, "metrics": { "model_steps": { "n": 219, "mean": 4.757990867579909, "median": 4.0, "p90": 6.0 }, "actions": { "n": 219, "mean": 4.639269406392694, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 219, "mean": 4.6255707762557075, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 219, "mean": 269.47250624148285, "median": 256.75, "p90": 351.05 }, "generated_tokens": { "n": 219, "mean": 1269.3287671232877, "median": 1145.0, "p90": 1758.2000000000003 }, "first_turn_tokens": { "n": 219, "mean": 224.3881278538813, "median": 222.0, "p90": 283.0 }, "searches": { "n": 219, "mean": 1.08675799086758, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 219, "mean": 1.082191780821918, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 219, "mean": 1.0776255707762556, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 219, "mean": 1.0593607305936072, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 219, "mean": 0.0182648401826484, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 219, "mean": 0.0867579908675799, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 219, "mean": 0.0319634703196347, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 219, "mean": 1.3424657534246576, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 219, "mean": 0.0228310502283105, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 219, "mean": 0.1187214611872146, "median": 0.0, "p90": 0.20000000000001705 }, "invalid_actions": { "n": 219, "mean": 0.0136986301369863, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 219, "mean": 344.07263766838713, "median": 328.75, "p90": 454.04 }, "first_product_rank": { "n": 219, "mean": 2.867579908675799, "median": 1.0, "p90": 7.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 266.77831094049907, "token_complete_episodes": 219, "rates": { "protocol_errors": 0.1004566210045662, "product_revisits": 0.0182648401826484, "option_changes": 0.0182648401826484, "backtracks": 0.0639269406392694 }, "terminations": { "purchase": 219 } }, "failure": { "n": 37, "tasks": 13, "metrics": { "model_steps": { "n": 37, "mean": 14.972972972972974, "median": 13.0, "p90": 30.0 }, "actions": { "n": 37, "mean": 10.45945945945946, "median": 6.0, "p90": 24.0 }, "valid_actions": { "n": 37, "mean": 7.675675675675675, "median": 5.0, "p90": 15.399999999999999 }, "tokens_per_turn": { "n": 37, "mean": 263.8938626104415, "median": 272.75, "p90": 332.25 }, "generated_tokens": { "n": 37, "mean": 3561.0810810810813, "median": 2965.0, "p90": 6435.999999999999 }, "first_turn_tokens": { "n": 37, "mean": 217.02702702702703, "median": 209.0, "p90": 332.79999999999995 }, "searches": { "n": 37, "mean": 2.1621621621621623, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 37, "mean": 2.108108108108108, "median": 1.0, "p90": 4.399999999999999 }, "product_opens": { "n": 37, "mean": 1.837837837837838, "median": 1.0, "p90": 4.399999999999999 }, "unique_products": { "n": 37, "mean": 1.3783783783783783, "median": 1.0, "p90": 2.799999999999997 }, "product_revisits": { "n": 37, "mean": 0.4594594594594595, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 37, "mean": 1.162162162162162, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 37, "mean": 0.5405405405405406, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 37, "mean": 1.4324324324324325, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 37, "mean": 0.2972972972972973, "median": 0.0, "p90": 0.3999999999999986 }, "protocol_errors": { "n": 37, "mean": 4.243243243243243, "median": 0.0, "p90": 29.0 }, "invalid_actions": { "n": 37, "mean": 2.7837837837837838, "median": 0.0, "p90": 10.0 }, "reasoning_chars_per_turn": { "n": 37, "mean": 361.39037505405923, "median": 367.0, "p90": 467.53999999999996 }, "first_product_rank": { "n": 30, "mean": 4.033333333333333, "median": 1.0, "p90": 12.100000000000001 } }, "success_rate": 0.0, "reward_mean": 0.036036036036036036, "pooled_tokens_per_turn": 237.83393501805054, "token_complete_episodes": 37, "rates": { "protocol_errors": 0.43243243243243246, "product_revisits": 0.24324324324324326, "option_changes": 0.10810810810810811, "backtracks": 0.40540540540540543 }, "terminations": { "purchase": 20, "action_limit": 7, "generation_length": 10 } } }, { "step": 22, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.1171875, "median": 4.0, "p90": 11.0 }, "actions": { "n": 256, "mean": 5.390625, "median": 4.0, "p90": 9.0 }, "valid_actions": { "n": 256, "mean": 5.26171875, "median": 4.0, "p90": 8.5 }, "tokens_per_turn": { "n": 256, "mean": 287.36647838241083, "median": 277.5, "p90": 379.125 }, "generated_tokens": { "n": 256, "mean": 1788.38671875, "median": 1291.5, "p90": 2979.5 }, "first_turn_tokens": { "n": 256, "mean": 203.67578125, "median": 198.0, "p90": 270.0 }, "searches": { "n": 256, "mean": 1.2890625, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.2890625, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.203125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.1328125, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.0703125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.2890625, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.140625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.4140625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.671875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.12890625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 376.1400330175943, "median": 362.425, "p90": 498.25 }, "first_product_rank": { "n": 248, "mean": 3.100806451612903, "median": 1.0, "p90": 7.300000000000011 } }, "success_rate": 0.72265625, "reward_mean": 0.7371651785714286, "pooled_tokens_per_turn": 292.3544061302682, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0859375, "product_revisits": 0.04296875, "option_changes": 0.0234375, "backtracks": 0.16015625 }, "terminations": { "purchase": 237, "action_limit": 5, "generation_length": 14 } }, "success": { "n": 185, "tasks": 30, "metrics": { "model_steps": { "n": 185, "mean": 4.870270270270271, "median": 4.0, "p90": 6.599999999999994 }, "actions": { "n": 185, "mean": 4.8, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 185, "mean": 4.767567567567568, "median": 4.0, "p90": 5.599999999999994 }, "tokens_per_turn": { "n": 185, "mean": 275.5542613392613, "median": 267.4, "p90": 354.825 }, "generated_tokens": { "n": 185, "mean": 1341.4162162162163, "median": 1232.0, "p90": 1886.4 }, "first_turn_tokens": { "n": 185, "mean": 207.55135135135134, "median": 203.0, "p90": 264.0 }, "searches": { "n": 185, "mean": 1.1135135135135135, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 185, "mean": 1.1135135135135135, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 185, "mean": 1.1135135135135135, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 185, "mean": 1.0864864864864865, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 185, "mean": 0.02702702702702703, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 185, "mean": 0.11351351351351352, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 185, "mean": 0.02702702702702703, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 185, "mean": 1.4, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 185, "mean": 0.010810810810810811, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 185, "mean": 0.07027027027027027, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 185, "mean": 0.032432432432432434, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 185, "mean": 356.73673862173865, "median": 352.0, "p90": 457.5 }, "first_product_rank": { "n": 185, "mean": 3.0216216216216214, "median": 1.0, "p90": 7.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 275.42952275249723, "token_complete_episodes": 185, "rates": { "protocol_errors": 0.03783783783783784, "product_revisits": 0.021621621621621623, "option_changes": 0.005405405405405406, "backtracks": 0.0918918918918919 }, "terminations": { "purchase": 185 } }, "failure": { "n": 71, "tasks": 18, "metrics": { "model_steps": { "n": 71, "mean": 9.366197183098592, "median": 5.0, "p90": 20.0 }, "actions": { "n": 71, "mean": 6.929577464788732, "median": 5.0, "p90": 15.0 }, "valid_actions": { "n": 71, "mean": 6.549295774647887, "median": 4.0, "p90": 13.0 }, "tokens_per_turn": { "n": 71, "mean": 318.14479039625115, "median": 307.25, "p90": 457.5 }, "generated_tokens": { "n": 71, "mean": 2953.0281690140846, "median": 1809.0, "p90": 6661.0 }, "first_turn_tokens": { "n": 71, "mean": 193.57746478873239, "median": 182.0, "p90": 271.0 }, "searches": { "n": 71, "mean": 1.7464788732394365, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 71, "mean": 1.7464788732394365, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 71, "mean": 1.4366197183098592, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 71, "mean": 1.2535211267605635, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 71, "mean": 0.18309859154929578, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 71, "mean": 0.7464788732394366, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 71, "mean": 0.43661971830985913, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 71, "mean": 1.4507042253521127, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 71, "mean": 0.07042253521126761, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 71, "mean": 2.23943661971831, "median": 0.0, "p90": 2.0 }, "invalid_actions": { "n": 71, "mean": 0.38028169014084506, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 71, "mean": 426.6979127814437, "median": 404.8125, "p90": 613.5333333333333 }, "first_product_rank": { "n": 63, "mean": 3.3333333333333335, "median": 2.0, "p90": 8.0 } }, "success_rate": 0.0, "reward_mean": 0.052313883299798795, "pooled_tokens_per_turn": 315.2857142857143, "token_complete_episodes": 71, "rates": { "protocol_errors": 0.2112676056338028, "product_revisits": 0.09859154929577464, "option_changes": 0.07042253521126761, "backtracks": 0.3380281690140845 }, "terminations": { "action_limit": 5, "purchase": 52, "generation_length": 14 } } }, { "step": 23, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.59765625, "median": 4.0, "p90": 14.5 }, "actions": { "n": 256, "mean": 6.08984375, "median": 4.0, "p90": 12.0 }, "valid_actions": { "n": 256, "mean": 5.42578125, "median": 4.0, "p90": 9.5 }, "tokens_per_turn": { "n": 256, "mean": 265.32820693903324, "median": 261.525, "p90": 335.5 }, "generated_tokens": { "n": 256, "mean": 1646.76953125, "median": 1165.0, "p90": 3928.5 }, "first_turn_tokens": { "n": 256, "mean": 209.81640625, "median": 202.5, "p90": 273.0 }, "searches": { "n": 256, "mean": 1.4140625, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 256, "mean": 1.41015625, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 256, "mean": 1.35546875, "median": 1.0, "p90": 2.5 }, "unique_products": { "n": 256, "mean": 1.125, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.23046875, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.42578125, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 256, "mean": 0.0546875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.26171875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.43359375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.6640625, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 347.1980239593165, "median": 336.125, "p90": 453.75 }, "first_product_rank": { "n": 253, "mean": 2.3043478260869565, "median": 1.0, "p90": 5.0 } }, "success_rate": 0.77734375, "reward_mean": 0.7799479166666667, "pooled_tokens_per_turn": 249.599171107164, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0703125, "product_revisits": 0.10546875, "option_changes": 0.00390625, "backtracks": 0.1875 }, "terminations": { "purchase": 234, "generation_length": 19, "action_limit": 3 } }, "success": { "n": 199, "tasks": 27, "metrics": { "model_steps": { "n": 199, "mean": 4.472361809045226, "median": 4.0, "p90": 5.0 }, "actions": { "n": 199, "mean": 4.442211055276382, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 199, "mean": 4.432160804020101, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 199, "mean": 265.525787361717, "median": 261.25, "p90": 334.05 }, "generated_tokens": { "n": 199, "mean": 1191.9547738693468, "median": 1102.0, "p90": 1512.0 }, "first_turn_tokens": { "n": 199, "mean": 206.66834170854273, "median": 201.0, "p90": 268.80000000000007 }, "searches": { "n": 199, "mean": 1.0804020100502512, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 199, "mean": 1.0804020100502512, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 199, "mean": 1.07035175879397, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 199, "mean": 1.0603015075376885, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 199, "mean": 0.010050251256281407, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 199, "mean": 0.08040201005025126, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 199, "mean": 0.02512562814070352, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 199, "mean": 1.1758793969849246, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 199, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 199, "mean": 0.03015075376884422, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 199, "mean": 0.010050251256281407, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 199, "mean": 345.7709838876924, "median": 336.25, "p90": 443.5 }, "first_product_rank": { "n": 199, "mean": 1.9045226130653266, "median": 1.0, "p90": 4.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 266.51573033707865, "token_complete_episodes": 199, "rates": { "protocol_errors": 0.020100502512562814, "product_revisits": 0.010050251256281407, "option_changes": 0.0, "backtracks": 0.07035175879396985 }, "terminations": { "purchase": 199 } }, "failure": { "n": 57, "tasks": 13, "metrics": { "model_steps": { "n": 57, "mean": 14.017543859649123, "median": 12.0, "p90": 27.4 }, "actions": { "n": 57, "mean": 11.842105263157896, "median": 9.0, "p90": 24.0 }, "valid_actions": { "n": 57, "mean": 8.894736842105264, "median": 9.0, "p90": 16.0 }, "tokens_per_turn": { "n": 57, "mean": 264.63840862124243, "median": 273.0, "p90": 346.5 }, "generated_tokens": { "n": 57, "mean": 3234.6315789473683, "median": 3600.0, "p90": 5335.8 }, "first_turn_tokens": { "n": 57, "mean": 220.80701754385964, "median": 217.0, "p90": 288.8 }, "searches": { "n": 57, "mean": 2.5789473684210527, "median": 2.0, "p90": 5.0 }, "unique_queries": { "n": 57, "mean": 2.56140350877193, "median": 2.0, "p90": 5.0 }, "product_opens": { "n": 57, "mean": 2.3508771929824563, "median": 2.0, "p90": 4.399999999999999 }, "unique_products": { "n": 57, "mean": 1.3508771929824561, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 57, "mean": 1.0, "median": 0.0, "p90": 3.0 }, "backtracks": { "n": 57, "mean": 1.631578947368421, "median": 1.0, "p90": 4.0 }, "pagination": { "n": 57, "mean": 0.15789473684210525, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 57, "mean": 1.5614035087719298, "median": 1.0, "p90": 3.3999999999999986 }, "option_changes": { "n": 57, "mean": 0.07017543859649122, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 57, "mean": 1.8421052631578947, "median": 0.0, "p90": 2.0 }, "invalid_actions": { "n": 57, "mean": 2.9473684210526314, "median": 0.0, "p90": 9.399999999999999 }, "reasoning_chars_per_turn": { "n": 57, "mean": 352.1801463146356, "median": 336.0, "p90": 465.85 }, "first_product_rank": { "n": 54, "mean": 3.7777777777777777, "median": 1.0, "p90": 9.400000000000006 } }, "success_rate": 0.0, "reward_mean": 0.011695906432748537, "pooled_tokens_per_turn": 230.75594493116395, "token_complete_episodes": 57, "rates": { "protocol_errors": 0.24561403508771928, "product_revisits": 0.43859649122807015, "option_changes": 0.017543859649122806, "backtracks": 0.5964912280701754 }, "terminations": { "purchase": 35, "generation_length": 19, "action_limit": 3 } } }, { "step": 24, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.68359375, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.55078125, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.546875, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 275.7551979758523, "median": 265.375, "p90": 362.375 }, "generated_tokens": { "n": 256, "mean": 1327.9609375, "median": 1097.5, "p90": 1907.0 }, "first_turn_tokens": { "n": 256, "mean": 201.44140625, "median": 196.0, "p90": 261.0 }, "searches": { "n": 256, "mean": 1.09765625, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.09765625, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.09375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.078125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.09765625, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.25390625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.1328125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 357.5315560036752, "median": 340.75, "p90": 480.25 }, "first_product_rank": { "n": 255, "mean": 1.9333333333333333, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.8125, "reward_mean": 0.8203125, "pooled_tokens_per_turn": 283.534612176814, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0234375, "product_revisits": 0.015625, "option_changes": 0.02734375, "backtracks": 0.078125 }, "terminations": { "purchase": 255, "action_limit": 1 } }, "success": { "n": 208, "tasks": 28, "metrics": { "model_steps": { "n": 208, "mean": 4.4375, "median": 4.0, "p90": 5.0 }, "actions": { "n": 208, "mean": 4.423076923076923, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 208, "mean": 4.418269230769231, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 208, "mean": 270.39463748057494, "median": 260.0, "p90": 360.01 }, "generated_tokens": { "n": 208, "mean": 1219.2740384615386, "median": 1083.5, "p90": 1591.4 }, "first_turn_tokens": { "n": 208, "mean": 201.06730769230768, "median": 194.5, "p90": 262.3 }, "searches": { "n": 208, "mean": 1.0625, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 208, "mean": 1.0625, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 208, "mean": 1.0625, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 208, "mean": 1.0576923076923077, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 208, "mean": 0.004807692307692308, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 208, "mean": 0.0625, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 208, "mean": 0.004807692307692308, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 208, "mean": 1.2259615384615385, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 208, "mean": 0.028846153846153848, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 208, "mean": 0.014423076923076924, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 208, "mean": 0.004807692307692308, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 208, "mean": 350.1343443015318, "median": 337.875, "p90": 478.65000000000003 }, "first_product_rank": { "n": 208, "mean": 1.6682692307692308, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 274.76598049837486, "token_complete_episodes": 208, "rates": { "protocol_errors": 0.014423076923076924, "product_revisits": 0.004807692307692308, "option_changes": 0.02403846153846154, "backtracks": 0.052884615384615384 }, "terminations": { "purchase": 208 } }, "failure": { "n": 48, "tasks": 10, "metrics": { "model_steps": { "n": 48, "mean": 5.75, "median": 4.0, "p90": 9.0 }, "actions": { "n": 48, "mean": 5.104166666666667, "median": 4.0, "p90": 8.300000000000004 }, "valid_actions": { "n": 48, "mean": 5.104166666666667, "median": 4.0, "p90": 8.300000000000004 }, "tokens_per_turn": { "n": 48, "mean": 298.9842934553872, "median": 291.4431818181818, "p90": 367.67500000000007 }, "generated_tokens": { "n": 48, "mean": 1798.9375, "median": 1289.0, "p90": 2873.800000000001 }, "first_turn_tokens": { "n": 48, "mean": 203.0625, "median": 200.0, "p90": 247.60000000000002 }, "searches": { "n": 48, "mean": 1.25, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 48, "mean": 1.25, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 48, "mean": 1.2291666666666667, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 48, "mean": 1.1666666666666667, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 48, "mean": 0.0625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 48, "mean": 0.25, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 48, "mean": 0.020833333333333332, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 48, "mean": 1.375, "median": 1.0, "p90": 2.3000000000000043 }, "option_changes": { "n": 48, "mean": 0.0625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 48, "mean": 0.6458333333333334, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 48, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 48, "mean": 389.58614004629635, "median": 388.0, "p90": 482.52500000000003 }, "first_product_rank": { "n": 47, "mean": 3.106382978723404, "median": 2.0, "p90": 3.799999999999997 } }, "success_rate": 0.0, "reward_mean": 0.041666666666666664, "pooled_tokens_per_turn": 312.85869565217394, "token_complete_episodes": 48, "rates": { "protocol_errors": 0.0625, "product_revisits": 0.0625, "option_changes": 0.041666666666666664, "backtracks": 0.1875 }, "terminations": { "purchase": 47, "action_limit": 1 } } }, { "step": 25, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.6015625, "median": 4.0, "p90": 10.0 }, "actions": { "n": 256, "mean": 5.55078125, "median": 4.0, "p90": 10.0 }, "valid_actions": { "n": 256, "mean": 5.51171875, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 256, "mean": 270.515095431241, "median": 256.125, "p90": 364.1071428571429 }, "generated_tokens": { "n": 256, "mean": 1554.10546875, "median": 1114.0, "p90": 3096.5 }, "first_turn_tokens": { "n": 256, "mean": 193.3359375, "median": 187.0, "p90": 255.0 }, "searches": { "n": 256, "mean": 1.25, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.2421875, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.19921875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.12109375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.25, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.08203125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.75390625, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.35546875, "median": 0.0, "p90": 0.5 }, "protocol_errors": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.0390625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 358.0295602230711, "median": 339.375, "p90": 503.45 }, "first_product_rank": { "n": 254, "mean": 2.1377952755905514, "median": 1.0, "p90": 4.700000000000017 } }, "success_rate": 0.8203125, "reward_mean": 0.8268229166666666, "pooled_tokens_per_turn": 277.44142259414224, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0234375, "product_revisits": 0.05859375, "option_changes": 0.1015625, "backtracks": 0.1328125 }, "terminations": { "purchase": 250, "generation_length": 6 } }, "success": { "n": 210, "tasks": 30, "metrics": { "model_steps": { "n": 210, "mean": 4.9904761904761905, "median": 4.0, "p90": 7.0 }, "actions": { "n": 210, "mean": 4.976190476190476, "median": 4.0, "p90": 7.0 }, "valid_actions": { "n": 210, "mean": 4.947619047619048, "median": 4.0, "p90": 7.0 }, "tokens_per_turn": { "n": 210, "mean": 256.99592806951506, "median": 245.625, "p90": 349.85 }, "generated_tokens": { "n": 210, "mean": 1309.3857142857144, "median": 1033.0, "p90": 2069.8999999999996 }, "first_turn_tokens": { "n": 210, "mean": 193.16666666666666, "median": 187.5, "p90": 256.1 }, "searches": { "n": 210, "mean": 1.119047619047619, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 210, "mean": 1.1095238095238096, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 210, "mean": 1.119047619047619, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 210, "mean": 1.0761904761904761, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 210, "mean": 0.04285714285714286, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 210, "mean": 0.11904761904761904, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 210, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 210, "mean": 1.5904761904761904, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 210, "mean": 0.26666666666666666, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 210, "mean": 0.014285714285714285, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 210, "mean": 0.02857142857142857, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 210, "mean": 336.8548756999505, "median": 315.75, "p90": 486.275 }, "first_product_rank": { "n": 210, "mean": 1.6476190476190475, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 262.3769083969466, "token_complete_episodes": 210, "rates": { "protocol_errors": 0.014285714285714285, "product_revisits": 0.04285714285714286, "option_changes": 0.09047619047619047, "backtracks": 0.08571428571428572 }, "terminations": { "purchase": 210 } }, "failure": { "n": 46, "tasks": 11, "metrics": { "model_steps": { "n": 46, "mean": 8.391304347826088, "median": 5.0, "p90": 16.5 }, "actions": { "n": 46, "mean": 8.173913043478262, "median": 5.0, "p90": 16.0 }, "valid_actions": { "n": 46, "mean": 8.08695652173913, "median": 5.0, "p90": 16.0 }, "tokens_per_turn": { "n": 46, "mean": 332.2330333869462, "median": 331.9375, "p90": 429.2857142857143 }, "generated_tokens": { "n": 46, "mean": 2671.304347826087, "median": 1819.0, "p90": 5170.0 }, "first_turn_tokens": { "n": 46, "mean": 194.1086956521739, "median": 185.5, "p90": 249.0 }, "searches": { "n": 46, "mean": 1.8478260869565217, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 46, "mean": 1.8478260869565217, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 46, "mean": 1.565217391304348, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 46, "mean": 1.326086956521739, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 46, "mean": 0.2391304347826087, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 46, "mean": 0.8478260869565217, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 46, "mean": 0.45652173913043476, "median": 0.0, "p90": 1.5 }, "option_selections": { "n": 46, "mean": 2.5, "median": 1.5, "p90": 4.5 }, "option_changes": { "n": 46, "mean": 0.7608695652173914, "median": 0.0, "p90": 1.0 }, "protocol_errors": { "n": 46, "mean": 0.08695652173913043, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 46, "mean": 0.08695652173913043, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 46, "mean": 454.6965982634037, "median": 454.90625, "p90": 579.825 }, "first_product_rank": { "n": 44, "mean": 4.4772727272727275, "median": 1.0, "p90": 12.0 } }, "success_rate": 0.0, "reward_mean": 0.03623188405797101, "pooled_tokens_per_turn": 318.3419689119171, "token_complete_episodes": 46, "rates": { "protocol_errors": 0.06521739130434782, "product_revisits": 0.13043478260869565, "option_changes": 0.15217391304347827, "backtracks": 0.34782608695652173 }, "terminations": { "purchase": 40, "generation_length": 6 } } }, { "step": 26, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.94140625, "median": 4.0, "p90": 11.0 }, "actions": { "n": 256, "mean": 5.7890625, "median": 4.0, "p90": 11.0 }, "valid_actions": { "n": 256, "mean": 5.74609375, "median": 4.0, "p90": 11.0 }, "tokens_per_turn": { "n": 256, "mean": 259.51187932006394, "median": 252.5, "p90": 331.0 }, "generated_tokens": { "n": 256, "mean": 1556.89453125, "median": 1098.0, "p90": 3256.0 }, "first_turn_tokens": { "n": 256, "mean": 190.19140625, "median": 187.0, "p90": 243.0 }, "searches": { "n": 256, "mean": 1.3671875, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.3671875, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.26953125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.1640625, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.10546875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.3671875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.1328125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.671875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.27734375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.08984375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 339.1964082509495, "median": 328.8, "p90": 448.125 }, "first_product_rank": { "n": 254, "mean": 1.5826771653543308, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.8125, "reward_mean": 0.8140625, "pooled_tokens_per_turn": 262.0414201183432, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.07421875, "product_revisits": 0.08203125, "option_changes": 0.0546875, "backtracks": 0.16796875 }, "terminations": { "purchase": 240, "generation_length": 16 } }, "success": { "n": 208, "tasks": 29, "metrics": { "model_steps": { "n": 208, "mean": 4.759615384615385, "median": 4.0, "p90": 5.0 }, "actions": { "n": 208, "mean": 4.716346153846154, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 208, "mean": 4.711538461538462, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 208, "mean": 254.913427618055, "median": 247.225, "p90": 328.40000000000003 }, "generated_tokens": { "n": 208, "mean": 1235.25, "median": 1042.0, "p90": 1625.9 }, "first_turn_tokens": { "n": 208, "mean": 192.46153846153845, "median": 190.0, "p90": 243.3 }, "searches": { "n": 208, "mean": 1.1201923076923077, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 208, "mean": 1.1201923076923077, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 208, "mean": 1.0865384615384615, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 208, "mean": 1.0528846153846154, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 208, "mean": 0.03365384615384615, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 208, "mean": 0.1201923076923077, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 208, "mean": 0.057692307692307696, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 208, "mean": 1.3269230769230769, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 208, "mean": 0.07692307692307693, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 208, "mean": 0.04326923076923077, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 208, "mean": 0.004807692307692308, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 208, "mean": 333.5136277984836, "median": 324.0, "p90": 444.665 }, "first_product_rank": { "n": 208, "mean": 1.4951923076923077, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 259.5272727272727, "token_complete_episodes": 208, "rates": { "protocol_errors": 0.03365384615384615, "product_revisits": 0.03365384615384615, "option_changes": 0.009615384615384616, "backtracks": 0.08653846153846154 }, "terminations": { "purchase": 208 } }, "failure": { "n": 48, "tasks": 12, "metrics": { "model_steps": { "n": 48, "mean": 11.0625, "median": 8.0, "p90": 22.300000000000004 }, "actions": { "n": 48, "mean": 10.4375, "median": 8.0, "p90": 21.0 }, "valid_actions": { "n": 48, "mean": 10.229166666666666, "median": 8.0, "p90": 20.300000000000004 }, "tokens_per_turn": { "n": 48, "mean": 279.43850336210267, "median": 275.0, "p90": 361.69473684210527 }, "generated_tokens": { "n": 48, "mean": 2950.6875, "median": 2109.0, "p90": 5838.500000000001 }, "first_turn_tokens": { "n": 48, "mean": 180.35416666666666, "median": 171.0, "p90": 233.3 }, "searches": { "n": 48, "mean": 2.4375, "median": 2.0, "p90": 5.0 }, "unique_queries": { "n": 48, "mean": 2.4375, "median": 2.0, "p90": 5.0 }, "product_opens": { "n": 48, "mean": 2.0625, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 48, "mean": 1.6458333333333333, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 48, "mean": 0.4166666666666667, "median": 0.0, "p90": 1.3000000000000043 }, "backtracks": { "n": 48, "mean": 1.4375, "median": 1.0, "p90": 4.0 }, "pagination": { "n": 48, "mean": 0.4583333333333333, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 48, "mean": 3.1666666666666665, "median": 2.0, "p90": 8.0 }, "option_changes": { "n": 48, "mean": 1.1458333333333333, "median": 0.0, "p90": 6.0 }, "protocol_errors": { "n": 48, "mean": 0.2916666666666667, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 48, "mean": 0.20833333333333334, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 48, "mean": 363.82179021163506, "median": 357.83333333333337, "p90": 485.2934210526317 }, "first_product_rank": { "n": 46, "mean": 1.9782608695652173, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.0, "reward_mean": 0.008333333333333333, "pooled_tokens_per_turn": 266.728813559322, "token_complete_episodes": 48, "rates": { "protocol_errors": 0.25, "product_revisits": 0.2916666666666667, "option_changes": 0.25, "backtracks": 0.5208333333333334 }, "terminations": { "purchase": 32, "generation_length": 16 } } }, { "step": 27, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.3515625, "median": 4.0, "p90": 8.0 }, "actions": { "n": 256, "mean": 5.26953125, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.234375, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 269.83606087361375, "median": 259.375, "p90": 349.0 }, "generated_tokens": { "n": 256, "mean": 1445.76171875, "median": 1113.0, "p90": 2236.5 }, "first_turn_tokens": { "n": 256, "mean": 208.69140625, "median": 198.0, "p90": 286.5 }, "searches": { "n": 256, "mean": 1.28125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.2734375, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.2421875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.140625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.1015625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.28125, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.4140625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0546875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 350.0882364129819, "median": 337.35, "p90": 457.5 }, "first_product_rank": { "n": 256, "mean": 2.171875, "median": 1.0, "p90": 5.0 } }, "success_rate": 0.921875, "reward_mean": 0.921875, "pooled_tokens_per_turn": 270.1569343065693, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0390625, "product_revisits": 0.0625, "option_changes": 0.01171875, "backtracks": 0.11328125 }, "terminations": { "purchase": 249, "generation_length": 7 } }, "success": { "n": 236, "tasks": 31, "metrics": { "model_steps": { "n": 236, "mean": 4.572033898305085, "median": 4.0, "p90": 5.0 }, "actions": { "n": 236, "mean": 4.555084745762712, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 236, "mean": 4.546610169491525, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 236, "mean": 269.57175068928245, "median": 259.375, "p90": 347.375 }, "generated_tokens": { "n": 236, "mean": 1251.6737288135594, "median": 1091.5, "p90": 1605.0 }, "first_turn_tokens": { "n": 236, "mean": 206.83050847457628, "median": 198.0, "p90": 283.5 }, "searches": { "n": 236, "mean": 1.0805084745762712, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 236, "mean": 1.0805084745762712, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 236, "mean": 1.0677966101694916, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 236, "mean": 1.0466101694915255, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 236, "mean": 0.0211864406779661, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 236, "mean": 0.08050847457627118, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 236, "mean": 0.025423728813559324, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 236, "mean": 1.2923728813559323, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 236, "mean": 0.0211864406779661, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 236, "mean": 0.01694915254237288, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 236, "mean": 0.00847457627118644, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 236, "mean": 349.07093456802147, "median": 337.35, "p90": 453.625 }, "first_product_rank": { "n": 236, "mean": 1.8601694915254237, "median": 1.0, "p90": 5.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 273.76737720111214, "token_complete_episodes": 236, "rates": { "protocol_errors": 0.01694915254237288, "product_revisits": 0.0211864406779661, "option_changes": 0.00847457627118644, "backtracks": 0.05084745762711865 }, "terminations": { "purchase": 236 } }, "failure": { "n": 20, "tasks": 7, "metrics": { "model_steps": { "n": 20, "mean": 14.55, "median": 16.0, "p90": 22.200000000000003 }, "actions": { "n": 20, "mean": 13.7, "median": 15.0, "p90": 21.0 }, "valid_actions": { "n": 20, "mean": 13.35, "median": 14.5, "p90": 20.0 }, "tokens_per_turn": { "n": 20, "mean": 272.95492104872244, "median": 260.20833333333337, "p90": 359.70625 }, "generated_tokens": { "n": 20, "mean": 3736.0, "median": 4218.5, "p90": 5309.7 }, "first_turn_tokens": { "n": 20, "mean": 230.65, "median": 210.0, "p90": 308.20000000000005 }, "searches": { "n": 20, "mean": 3.65, "median": 4.0, "p90": 6.0 }, "unique_queries": { "n": 20, "mean": 3.55, "median": 3.5, "p90": 6.0 }, "product_opens": { "n": 20, "mean": 3.3, "median": 3.0, "p90": 5.0 }, "unique_products": { "n": 20, "mean": 2.25, "median": 2.0, "p90": 3.1000000000000014 }, "product_revisits": { "n": 20, "mean": 1.05, "median": 1.0, "p90": 2.1000000000000014 }, "backtracks": { "n": 20, "mean": 2.65, "median": 3.0, "p90": 5.0 }, "pagination": { "n": 20, "mean": 0.25, "median": 0.0, "p90": 1.1000000000000014 }, "option_selections": { "n": 20, "mean": 2.85, "median": 3.0, "p90": 4.100000000000001 }, "option_changes": { "n": 20, "mean": 0.05, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 20, "mean": 0.5, "median": 0.0, "p90": 1.2000000000000028 }, "invalid_actions": { "n": 20, "mean": 0.35, "median": 0.0, "p90": 1.1000000000000014 }, "reasoning_chars_per_turn": { "n": 20, "mean": 362.0923981835159, "median": 339.375, "p90": 481.58125 }, "first_product_rank": { "n": 20, "mean": 5.85, "median": 1.0, "p90": 13.0 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 256.7697594501718, "token_complete_episodes": 20, "rates": { "protocol_errors": 0.3, "product_revisits": 0.55, "option_changes": 0.05, "backtracks": 0.85 }, "terminations": { "purchase": 13, "generation_length": 7 } } }, { "step": 28, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.49609375, "median": 4.0, "p90": 9.0 }, "actions": { "n": 256, "mean": 5.3984375, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.1875, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 273.8412896156984, "median": 265.625, "p90": 349.6019736842105 }, "generated_tokens": { "n": 256, "mean": 1548.81640625, "median": 1103.5, "p90": 3005.5 }, "first_turn_tokens": { "n": 256, "mean": 198.4921875, "median": 188.5, "p90": 260.5 }, "searches": { "n": 256, "mean": 1.3203125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.31640625, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.296875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.12890625, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.16796875, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.3203125, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.26171875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.2109375, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 355.9700140042498, "median": 349.0, "p90": 479.25 }, "first_product_rank": { "n": 256, "mean": 2.0546875, "median": 1.0, "p90": 3.5 } }, "success_rate": 0.875, "reward_mean": 0.875, "pooled_tokens_per_turn": 281.80312722103764, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0390625, "product_revisits": 0.109375, "option_changes": 0.0, "backtracks": 0.15625 }, "terminations": { "generation_length": 12, "purchase": 244 } }, "success": { "n": 224, "tasks": 30, "metrics": { "model_steps": { "n": 224, "mean": 4.424107142857143, "median": 4.0, "p90": 5.0 }, "actions": { "n": 224, "mean": 4.40625, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 224, "mean": 4.40625, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 224, "mean": 265.3311840503247, "median": 258.625, "p90": 344.75 }, "generated_tokens": { "n": 224, "mean": 1186.3214285714287, "median": 1060.5, "p90": 1509.0 }, "first_turn_tokens": { "n": 224, "mean": 199.69196428571428, "median": 190.5, "p90": 265.40000000000003 }, "searches": { "n": 224, "mean": 1.09375, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 224, "mean": 1.09375, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 224, "mean": 1.0892857142857142, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 224, "mean": 1.0491071428571428, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 224, "mean": 0.04017857142857143, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 224, "mean": 0.09375, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 224, "mean": 0.017857142857142856, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 224, "mean": 1.1116071428571428, "median": 1.0, "p90": 1.0 }, "option_changes": { "n": 224, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 224, "mean": 0.017857142857142856, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 224, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 224, "mean": 347.7707177804834, "median": 336.0, "p90": 464.25 }, "first_product_rank": { "n": 224, "mean": 2.0044642857142856, "median": 1.0, "p90": 3.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 268.14934409687186, "token_complete_episodes": 224, "rates": { "protocol_errors": 0.017857142857142856, "product_revisits": 0.04017857142857143, "option_changes": 0.0, "backtracks": 0.06696428571428571 }, "terminations": { "purchase": 224 } }, "failure": { "n": 32, "tasks": 10, "metrics": { "model_steps": { "n": 32, "mean": 13.0, "median": 14.0, "p90": 20.900000000000002 }, "actions": { "n": 32, "mean": 12.34375, "median": 13.5, "p90": 18.900000000000002 }, "valid_actions": { "n": 32, "mean": 10.65625, "median": 12.0, "p90": 16.0 }, "tokens_per_turn": { "n": 32, "mean": 333.4120285733144, "median": 305.325, "p90": 397.82500000000005 }, "generated_tokens": { "n": 32, "mean": 4086.28125, "median": 4206.5, "p90": 6277.0 }, "first_turn_tokens": { "n": 32, "mean": 190.09375, "median": 182.0, "p90": 246.8 }, "searches": { "n": 32, "mean": 2.90625, "median": 3.0, "p90": 4.0 }, "unique_queries": { "n": 32, "mean": 2.875, "median": 3.0, "p90": 4.0 }, "product_opens": { "n": 32, "mean": 2.75, "median": 3.0, "p90": 4.0 }, "unique_products": { "n": 32, "mean": 1.6875, "median": 2.0, "p90": 2.900000000000002 }, "product_revisits": { "n": 32, "mean": 1.0625, "median": 1.0, "p90": 2.0 }, "backtracks": { "n": 32, "mean": 1.90625, "median": 2.0, "p90": 3.0 }, "pagination": { "n": 32, "mean": 0.15625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 32, "mean": 2.3125, "median": 2.0, "p90": 4.0 }, "option_changes": { "n": 32, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 32, "mean": 0.28125, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 32, "mean": 1.6875, "median": 0.0, "p90": 6.0 }, "reasoning_chars_per_turn": { "n": 32, "mean": 413.3650875706144, "median": 411.29239766081866, "p90": 536.2625 }, "first_product_rank": { "n": 32, "mean": 2.40625, "median": 1.0, "p90": 5.900000000000002 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 314.3293269230769, "token_complete_episodes": 32, "rates": { "protocol_errors": 0.1875, "product_revisits": 0.59375, "option_changes": 0.0, "backtracks": 0.78125 }, "terminations": { "generation_length": 12, "purchase": 20 } } }, { "step": 29, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.51171875, "median": 4.0, "p90": 8.0 }, "actions": { "n": 256, "mean": 5.44921875, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.27734375, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 269.05905854132243, "median": 251.8, "p90": 365.975 }, "generated_tokens": { "n": 256, "mean": 1489.078125, "median": 1090.0, "p90": 2384.5 }, "first_turn_tokens": { "n": 256, "mean": 198.015625, "median": 185.0, "p90": 268.5 }, "searches": { "n": 256, "mean": 1.1796875, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.1796875, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.1484375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.078125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.0703125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.1796875, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.06640625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.75, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.40625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.171875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 341.5988910200043, "median": 317.175, "p90": 477.5 }, "first_product_rank": { "n": 254, "mean": 2.358267716535433, "median": 1.0, "p90": 5.0 } }, "success_rate": 0.83203125, "reward_mean": 0.8390625, "pooled_tokens_per_turn": 270.16583982990784, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.015625, "product_revisits": 0.046875, "option_changes": 0.09375, "backtracks": 0.09375 }, "terminations": { "purchase": 244, "generation_length": 12 } }, "success": { "n": 213, "tasks": 30, "metrics": { "model_steps": { "n": 213, "mean": 4.474178403755869, "median": 4.0, "p90": 5.0 }, "actions": { "n": 213, "mean": 4.469483568075117, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 213, "mean": 4.450704225352113, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 213, "mean": 261.453025847392, "median": 244.0, "p90": 364.1 }, "generated_tokens": { "n": 213, "mean": 1174.1784037558687, "median": 1041.0, "p90": 1686.4 }, "first_turn_tokens": { "n": 213, "mean": 197.9718309859155, "median": 185.0, "p90": 265.8 }, "searches": { "n": 213, "mean": 1.0375586854460095, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 213, "mean": 1.0375586854460095, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 213, "mean": 1.0328638497652582, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 213, "mean": 1.028169014084507, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 213, "mean": 0.004694835680751174, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 213, "mean": 0.03755868544600939, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 213, "mean": 0.014084507042253521, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 213, "mean": 1.3286384976525822, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 213, "mean": 0.10328638497652583, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 213, "mean": 0.004694835680751174, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 213, "mean": 0.018779342723004695, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 213, "mean": 335.9798920302442, "median": 309.8, "p90": 478.25 }, "first_product_rank": { "n": 213, "mean": 1.732394366197183, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 262.43441762854144, "token_complete_episodes": 213, "rates": { "protocol_errors": 0.004694835680751174, "product_revisits": 0.004694835680751174, "option_changes": 0.04225352112676056, "backtracks": 0.03286384976525822 }, "terminations": { "purchase": 213 } }, "failure": { "n": 43, "tasks": 13, "metrics": { "model_steps": { "n": 43, "mean": 10.651162790697674, "median": 8.0, "p90": 22.0 }, "actions": { "n": 43, "mean": 10.30232558139535, "median": 8.0, "p90": 20.800000000000004 }, "valid_actions": { "n": 43, "mean": 9.372093023255815, "median": 7.0, "p90": 19.0 }, "tokens_per_turn": { "n": 43, "mean": 306.73545304846624, "median": 271.6363636363636, "p90": 422.7466666666669 }, "generated_tokens": { "n": 43, "mean": 3048.9302325581393, "median": 2057.0, "p90": 6009.200000000001 }, "first_turn_tokens": { "n": 43, "mean": 198.2325581395349, "median": 186.0, "p90": 294.80000000000007 }, "searches": { "n": 43, "mean": 1.8837209302325582, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 43, "mean": 1.8837209302325582, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 43, "mean": 1.7209302325581395, "median": 1.0, "p90": 3.8000000000000043 }, "unique_products": { "n": 43, "mean": 1.3255813953488371, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 43, "mean": 0.3953488372093023, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 43, "mean": 0.8837209302325582, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 43, "mean": 0.32558139534883723, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 43, "mean": 3.8372093023255816, "median": 1.0, "p90": 11.0 }, "option_changes": { "n": 43, "mean": 1.9069767441860466, "median": 0.0, "p90": 7.800000000000004 }, "protocol_errors": { "n": 43, "mean": 0.06976744186046512, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 43, "mean": 0.9302325581395349, "median": 0.0, "p90": 4.200000000000017 }, "reasoning_chars_per_turn": { "n": 43, "mean": 369.43253717858374, "median": 367.0, "p90": 456.55916666666667 }, "first_product_rank": { "n": 41, "mean": 5.609756097560975, "median": 3.0, "p90": 16.0 } }, "success_rate": 0.0, "reward_mean": 0.04186046511627907, "pooled_tokens_per_turn": 286.2532751091703, "token_complete_episodes": 43, "rates": { "protocol_errors": 0.06976744186046512, "product_revisits": 0.2558139534883721, "option_changes": 0.3488372093023256, "backtracks": 0.3953488372093023 }, "terminations": { "purchase": 31, "generation_length": 12 } } }, { "step": 30, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.2578125, "median": 4.0, "p90": 8.5 }, "actions": { "n": 256, "mean": 5.2109375, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.1953125, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 270.5545332983431, "median": 264.625, "p90": 343.575 }, "generated_tokens": { "n": 256, "mean": 1414.7890625, "median": 1120.0, "p90": 2258.5 }, "first_turn_tokens": { "n": 256, "mean": 189.55859375, "median": 186.5, "p90": 246.5 }, "searches": { "n": 256, "mean": 1.23046875, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.2265625, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.20703125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.14453125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.0625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.23046875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.54296875, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.1640625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 352.49380577987006, "median": 341.625, "p90": 457.5333333333333 }, "first_product_rank": { "n": 256, "mean": 1.8125, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.83203125, "reward_mean": 0.8367660984848484, "pooled_tokens_per_turn": 269.0832095096583, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0234375, "product_revisits": 0.05078125, "option_changes": 0.0703125, "backtracks": 0.12109375 }, "terminations": { "purchase": 250, "generation_length": 6 } }, "success": { "n": 213, "tasks": 31, "metrics": { "model_steps": { "n": 213, "mean": 4.460093896713615, "median": 4.0, "p90": 5.0 }, "actions": { "n": 213, "mean": 4.455399061032864, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 213, "mean": 4.455399061032864, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 213, "mean": 269.64223184223187, "median": 260.5, "p90": 342.55 }, "generated_tokens": { "n": 213, "mean": 1193.5774647887324, "median": 1103.0, "p90": 1559.2000000000003 }, "first_turn_tokens": { "n": 213, "mean": 190.49295774647888, "median": 188.0, "p90": 249.0 }, "searches": { "n": 213, "mean": 1.0610328638497653, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 213, "mean": 1.056338028169014, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 213, "mean": 1.0610328638497653, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 213, "mean": 1.0422535211267605, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 213, "mean": 0.018779342723004695, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 213, "mean": 0.06103286384976526, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 213, "mean": 0.004694835680751174, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 213, "mean": 1.267605633802817, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 213, "mean": 0.03286384976525822, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 213, "mean": 0.004694835680751174, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 213, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 213, "mean": 352.0553397828046, "median": 337.0, "p90": 461.2 }, "first_product_rank": { "n": 213, "mean": 1.4929577464788732, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 267.61263157894734, "token_complete_episodes": 213, "rates": { "protocol_errors": 0.004694835680751174, "product_revisits": 0.018779342723004695, "option_changes": 0.03286384976525822, "backtracks": 0.051643192488262914 }, "terminations": { "purchase": 213 } }, "failure": { "n": 43, "tasks": 10, "metrics": { "model_steps": { "n": 43, "mean": 9.209302325581396, "median": 6.0, "p90": 19.0 }, "actions": { "n": 43, "mean": 8.953488372093023, "median": 6.0, "p90": 18.0 }, "valid_actions": { "n": 43, "mean": 8.86046511627907, "median": 6.0, "p90": 18.0 }, "tokens_per_turn": { "n": 43, "mean": 275.0736079530336, "median": 274.6666666666667, "p90": 342.17 }, "generated_tokens": { "n": 43, "mean": 2510.5581395348836, "median": 1735.0, "p90": 5153.600000000003 }, "first_turn_tokens": { "n": 43, "mean": 184.93023255813952, "median": 176.0, "p90": 230.00000000000006 }, "searches": { "n": 43, "mean": 2.0697674418604652, "median": 1.0, "p90": 4.800000000000004 }, "unique_queries": { "n": 43, "mean": 2.0697674418604652, "median": 1.0, "p90": 4.800000000000004 }, "product_opens": { "n": 43, "mean": 1.930232558139535, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 43, "mean": 1.6511627906976745, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 43, "mean": 0.27906976744186046, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 43, "mean": 1.069767441860465, "median": 0.0, "p90": 3.8000000000000043 }, "pagination": { "n": 43, "mean": 0.023255813953488372, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 43, "mean": 2.9069767441860463, "median": 2.0, "p90": 5.0 }, "option_changes": { "n": 43, "mean": 0.813953488372093, "median": 0.0, "p90": 2.0 }, "protocol_errors": { "n": 43, "mean": 0.11627906976744186, "median": 0.0, "p90": 0.8000000000000043 }, "invalid_actions": { "n": 43, "mean": 0.09302325581395349, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 43, "mean": 354.66574199789244, "median": 356.25, "p90": 451.08 }, "first_product_rank": { "n": 43, "mean": 3.395348837209302, "median": 1.0, "p90": 10.800000000000004 } }, "success_rate": 0.0, "reward_mean": 0.028188865398167718, "pooled_tokens_per_turn": 272.6111111111111, "token_complete_episodes": 43, "rates": { "protocol_errors": 0.11627906976744186, "product_revisits": 0.20930232558139536, "option_changes": 0.2558139534883721, "backtracks": 0.46511627906976744 }, "terminations": { "generation_length": 6, "purchase": 37 } } }, { "step": 31, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.53515625, "median": 4.0, "p90": 13.0 }, "actions": { "n": 256, "mean": 6.41015625, "median": 4.0, "p90": 12.0 }, "valid_actions": { "n": 256, "mean": 6.11328125, "median": 4.0, "p90": 12.0 }, "tokens_per_turn": { "n": 256, "mean": 248.32137927147505, "median": 240.22727272727275, "p90": 325.4642857142857 }, "generated_tokens": { "n": 256, "mean": 1573.51953125, "median": 1114.5, "p90": 3479.5 }, "first_turn_tokens": { "n": 256, "mean": 202.765625, "median": 199.0, "p90": 269.0 }, "searches": { "n": 256, "mean": 1.46484375, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 256, "mean": 1.4609375, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 256, "mean": 1.40625, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 256, "mean": 1.22265625, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.18359375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.47265625, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.80078125, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.1796875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.296875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 323.209971202651, "median": 309.975, "p90": 430.125 }, "first_product_rank": { "n": 255, "mean": 1.7019607843137254, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.66796875, "reward_mean": 0.6897631448412698, "pooled_tokens_per_turn": 240.77764494919307, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0546875, "product_revisits": 0.08984375, "option_changes": 0.0390625, "backtracks": 0.1796875 }, "terminations": { "purchase": 236, "generation_length": 16, "action_limit": 4 } }, "success": { "n": 171, "tasks": 24, "metrics": { "model_steps": { "n": 171, "mean": 4.988304093567251, "median": 4.0, "p90": 5.0 }, "actions": { "n": 171, "mean": 4.976608187134503, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 171, "mean": 4.976608187134503, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 171, "mean": 249.12151844257107, "median": 244.1, "p90": 325.5 }, "generated_tokens": { "n": 171, "mean": 1253.6374269005848, "median": 1063.0, "p90": 1654.0 }, "first_turn_tokens": { "n": 171, "mean": 209.69590643274853, "median": 207.0, "p90": 275.0 }, "searches": { "n": 171, "mean": 1.1637426900584795, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 171, "mean": 1.1578947368421053, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 171, "mean": 1.1578947368421053, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 171, "mean": 1.1169590643274854, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 171, "mean": 0.04093567251461988, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 171, "mean": 0.16374269005847952, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 171, "mean": 0.011695906432748537, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 171, "mean": 1.4795321637426901, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 171, "mean": 0.05847953216374269, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 171, "mean": 0.011695906432748537, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 171, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 171, "mean": 323.56961849067113, "median": 312.8, "p90": 431.5 }, "first_product_rank": { "n": 171, "mean": 1.1929824561403508, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 251.31535756154747, "token_complete_episodes": 171, "rates": { "protocol_errors": 0.011695906432748537, "product_revisits": 0.03508771929824561, "option_changes": 0.011695906432748537, "backtracks": 0.08187134502923976 }, "terminations": { "purchase": 171 } }, "failure": { "n": 85, "tasks": 14, "metrics": { "model_steps": { "n": 85, "mean": 9.647058823529411, "median": 5.0, "p90": 22.60000000000001 }, "actions": { "n": 85, "mean": 9.294117647058824, "median": 5.0, "p90": 21.0 }, "valid_actions": { "n": 85, "mean": 8.4, "median": 5.0, "p90": 19.60000000000001 }, "tokens_per_turn": { "n": 85, "mean": 246.7116875272701, "median": 236.2, "p90": 320.557142857143 }, "generated_tokens": { "n": 85, "mean": 2217.0470588235294, "median": 1253.0, "p90": 5077.600000000002 }, "first_turn_tokens": { "n": 85, "mean": 188.8235294117647, "median": 191.0, "p90": 247.40000000000003 }, "searches": { "n": 85, "mean": 2.070588235294118, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 85, "mean": 2.070588235294118, "median": 1.0, "p90": 5.0 }, "product_opens": { "n": 85, "mean": 1.9058823529411764, "median": 1.0, "p90": 5.0 }, "unique_products": { "n": 85, "mean": 1.4352941176470588, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 85, "mean": 0.47058823529411764, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 85, "mean": 1.0941176470588236, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 85, "mean": 0.11764705882352941, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 85, "mean": 2.447058823529412, "median": 2.0, "p90": 5.6000000000000085 }, "option_changes": { "n": 85, "mean": 0.4235294117647059, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 85, "mean": 0.16470588235294117, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 85, "mean": 0.8941176470588236, "median": 0.0, "p90": 1.6000000000000085 }, "reasoning_chars_per_turn": { "n": 85, "mean": 322.4864454820457, "median": 305.75, "p90": 425.75130434782614 }, "first_product_rank": { "n": 84, "mean": 2.738095238095238, "median": 1.0, "p90": 7.0 } }, "success_rate": 0.0, "reward_mean": 0.06563958916900094, "pooled_tokens_per_turn": 229.81585365853658, "token_complete_episodes": 85, "rates": { "protocol_errors": 0.1411764705882353, "product_revisits": 0.2, "option_changes": 0.09411764705882353, "backtracks": 0.3764705882352941 }, "terminations": { "purchase": 65, "generation_length": 16, "action_limit": 4 } } }, { "step": 32, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.0625, "median": 4.0, "p90": 10.0 }, "actions": { "n": 256, "mean": 5.94921875, "median": 4.0, "p90": 9.5 }, "valid_actions": { "n": 256, "mean": 5.4921875, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 256, "mean": 248.24320645668467, "median": 236.0, "p90": 326.0 }, "generated_tokens": { "n": 256, "mean": 1513.2890625, "median": 1046.0, "p90": 2997.5 }, "first_turn_tokens": { "n": 256, "mean": 190.40625, "median": 185.5, "p90": 252.5 }, "searches": { "n": 256, "mean": 1.30078125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.30078125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.26953125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.109375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.16015625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.30078125, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.640625, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.1796875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.45703125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 321.3652764447395, "median": 302.375, "p90": 430.75 }, "first_product_rank": { "n": 256, "mean": 1.68359375, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.8125, "reward_mean": 0.8125, "pooled_tokens_per_turn": 249.61469072164948, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.04296875, "product_revisits": 0.08203125, "option_changes": 0.0546875, "backtracks": 0.125 }, "terminations": { "purchase": 240, "generation_length": 16 } }, "success": { "n": 208, "tasks": 29, "metrics": { "model_steps": { "n": 208, "mean": 4.605769230769231, "median": 4.0, "p90": 5.0 }, "actions": { "n": 208, "mean": 4.591346153846154, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 208, "mean": 4.586538461538462, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 208, "mean": 242.72553352096617, "median": 232.17261904761904, "p90": 321.74 }, "generated_tokens": { "n": 208, "mean": 1114.0961538461538, "median": 1008.5, "p90": 1435.9 }, "first_turn_tokens": { "n": 208, "mean": 188.23557692307693, "median": 184.0, "p90": 248.3 }, "searches": { "n": 208, "mean": 1.0528846153846154, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 208, "mean": 1.0528846153846154, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 208, "mean": 1.0480769230769231, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 208, "mean": 1.0336538461538463, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 208, "mean": 0.014423076923076924, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 208, "mean": 0.052884615384615384, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 208, "mean": 0.019230769230769232, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 208, "mean": 1.4134615384615385, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 208, "mean": 0.03365384615384615, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 208, "mean": 0.014423076923076924, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 208, "mean": 0.004807692307692308, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 208, "mean": 313.7527491472203, "median": 295.875, "p90": 425.01 }, "first_product_rank": { "n": 208, "mean": 1.4615384615384615, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 241.89144050104383, "token_complete_episodes": 208, "rates": { "protocol_errors": 0.014423076923076924, "product_revisits": 0.014423076923076924, "option_changes": 0.028846153846153848, "backtracks": 0.03365384615384615 }, "terminations": { "purchase": 208 } }, "failure": { "n": 48, "tasks": 13, "metrics": { "model_steps": { "n": 48, "mean": 12.375, "median": 8.0, "p90": 23.0 }, "actions": { "n": 48, "mean": 11.833333333333334, "median": 8.0, "p90": 22.0 }, "valid_actions": { "n": 48, "mean": 9.416666666666666, "median": 8.0, "p90": 16.0 }, "tokens_per_turn": { "n": 48, "mean": 272.1531225114647, "median": 264.5, "p90": 349.10625000000005 }, "generated_tokens": { "n": 48, "mean": 3243.125, "median": 2494.0, "p90": 5528.1 }, "first_turn_tokens": { "n": 48, "mean": 199.8125, "median": 192.5, "p90": 264.50000000000006 }, "searches": { "n": 48, "mean": 2.375, "median": 2.0, "p90": 5.0 }, "unique_queries": { "n": 48, "mean": 2.375, "median": 2.0, "p90": 5.0 }, "product_opens": { "n": 48, "mean": 2.2291666666666665, "median": 2.0, "p90": 4.0 }, "unique_products": { "n": 48, "mean": 1.4375, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 48, "mean": 0.7916666666666666, "median": 0.0, "p90": 2.3000000000000043 }, "backtracks": { "n": 48, "mean": 1.375, "median": 1.0, "p90": 4.0 }, "pagination": { "n": 48, "mean": 0.14583333333333334, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 48, "mean": 2.625, "median": 2.0, "p90": 4.0 }, "option_changes": { "n": 48, "mean": 0.8125, "median": 0.0, "p90": 2.0 }, "protocol_errors": { "n": 48, "mean": 0.20833333333333334, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 48, "mean": 2.4166666666666665, "median": 0.0, "p90": 8.0 }, "reasoning_chars_per_turn": { "n": 48, "mean": 354.3528947339892, "median": 340.41776315789474, "p90": 470.49 }, "first_product_rank": { "n": 48, "mean": 2.6458333333333335, "median": 1.0, "p90": 2.6000000000000085 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 262.07070707070704, "token_complete_episodes": 48, "rates": { "protocol_errors": 0.16666666666666666, "product_revisits": 0.375, "option_changes": 0.16666666666666666, "backtracks": 0.5208333333333334 }, "terminations": { "purchase": 32, "generation_length": 16 } } }, { "step": 33, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.96875, "median": 4.0, "p90": 8.5 }, "actions": { "n": 256, "mean": 5.83203125, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.52734375, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 258.97834206800775, "median": 257.9666666666667, "p90": 323.875 }, "generated_tokens": { "n": 256, "mean": 1537.8359375, "median": 1104.5, "p90": 2409.0 }, "first_turn_tokens": { "n": 256, "mean": 204.8046875, "median": 205.0, "p90": 267.0 }, "searches": { "n": 256, "mean": 1.359375, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.359375, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.30859375, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.18359375, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.359375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.55859375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.1328125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.06640625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.3046875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 340.30702155445823, "median": 337.8, "p90": 433.25 }, "first_product_rank": { "n": 256, "mean": 1.81640625, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.828125, "reward_mean": 0.8302951388888888, "pooled_tokens_per_turn": 257.6479057591623, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0390625, "product_revisits": 0.07421875, "option_changes": 0.0546875, "backtracks": 0.12109375 }, "terminations": { "purchase": 238, "generation_length": 18 } }, "success": { "n": 212, "tasks": 29, "metrics": { "model_steps": { "n": 212, "mean": 4.44811320754717, "median": 4.0, "p90": 5.0 }, "actions": { "n": 212, "mean": 4.44811320754717, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 212, "mean": 4.44811320754717, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 212, "mean": 257.50194481880806, "median": 256.875, "p90": 323.575 }, "generated_tokens": { "n": 212, "mean": 1140.0377358490566, "median": 1078.5, "p90": 1517.8000000000002 }, "first_turn_tokens": { "n": 212, "mean": 203.18396226415095, "median": 201.0, "p90": 267.8 }, "searches": { "n": 212, "mean": 1.0188679245283019, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 212, "mean": 1.0188679245283019, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 212, "mean": 1.0188679245283019, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 212, "mean": 1.0188679245283019, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 212, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 212, "mean": 0.018867924528301886, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 212, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 212, "mean": 1.3915094339622642, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 212, "mean": 0.09433962264150944, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 212, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 212, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 212, "mean": 338.45929544773884, "median": 337.3, "p90": 431.175 }, "first_product_rank": { "n": 212, "mean": 1.3632075471698113, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 256.2969247083775, "token_complete_episodes": 212, "rates": { "protocol_errors": 0.0, "product_revisits": 0.0, "option_changes": 0.05188679245283019, "backtracks": 0.018867924528301886 }, "terminations": { "purchase": 212 } }, "failure": { "n": 44, "tasks": 9, "metrics": { "model_steps": { "n": 44, "mean": 13.295454545454545, "median": 12.0, "p90": 23.0 }, "actions": { "n": 44, "mean": 12.5, "median": 12.0, "p90": 21.700000000000003 }, "valid_actions": { "n": 44, "mean": 10.727272727272727, "median": 9.5, "p90": 21.0 }, "tokens_per_turn": { "n": 44, "mean": 266.09189245051556, "median": 263.1071428571429, "p90": 328.45000000000005 }, "generated_tokens": { "n": 44, "mean": 3454.5, "median": 4111.0, "p90": 6100.7 }, "first_turn_tokens": { "n": 44, "mean": 212.61363636363637, "median": 218.0, "p90": 258.7 }, "searches": { "n": 44, "mean": 3.0, "median": 2.5, "p90": 6.0 }, "unique_queries": { "n": 44, "mean": 3.0, "median": 2.5, "p90": 6.0 }, "product_opens": { "n": 44, "mean": 2.7045454545454546, "median": 2.0, "p90": 5.0 }, "unique_products": { "n": 44, "mean": 1.9772727272727273, "median": 2.0, "p90": 3.700000000000003 }, "product_revisits": { "n": 44, "mean": 0.7272727272727273, "median": 0.0, "p90": 3.0 }, "backtracks": { "n": 44, "mean": 2.0, "median": 1.5, "p90": 5.0 }, "pagination": { "n": 44, "mean": 0.06818181818181818, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 44, "mean": 2.3636363636363638, "median": 1.0, "p90": 5.0 }, "option_changes": { "n": 44, "mean": 0.3181818181818182, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 44, "mean": 0.38636363636363635, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 44, "mean": 1.7727272727272727, "median": 0.0, "p90": 7.0 }, "reasoning_chars_per_turn": { "n": 44, "mean": 349.20970188683333, "median": 353.1230158730159, "p90": 452.70000000000005 }, "first_product_rank": { "n": 44, "mean": 4.0, "median": 1.0, "p90": 11.400000000000006 } }, "success_rate": 0.0, "reward_mean": 0.012626262626262626, "pooled_tokens_per_turn": 259.825641025641, "token_complete_episodes": 44, "rates": { "protocol_errors": 0.22727272727272727, "product_revisits": 0.4318181818181818, "option_changes": 0.06818181818181818, "backtracks": 0.6136363636363636 }, "terminations": { "purchase": 26, "generation_length": 18 } } }, { "step": 34, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.109375, "median": 4.0, "p90": 8.0 }, "actions": { "n": 256, "mean": 5.0703125, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 4.99609375, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 249.1498216219034, "median": 235.0, "p90": 333.175 }, "generated_tokens": { "n": 256, "mean": 1280.4375, "median": 1061.5, "p90": 2052.5 }, "first_turn_tokens": { "n": 256, "mean": 193.1640625, "median": 194.0, "p90": 252.0 }, "searches": { "n": 256, "mean": 1.17578125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.171875, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.1640625, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.11328125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.1796875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.0390625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.44921875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.07421875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 325.325600414062, "median": 311.625, "p90": 438.75 }, "first_product_rank": { "n": 256, "mean": 2.15625, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.76171875, "reward_mean": 0.7734375, "pooled_tokens_per_turn": 250.60550458715596, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.015625, "product_revisits": 0.03515625, "option_changes": 0.03125, "backtracks": 0.109375 }, "terminations": { "purchase": 253, "generation_length": 3 } }, "success": { "n": 195, "tasks": 28, "metrics": { "model_steps": { "n": 195, "mean": 4.630769230769231, "median": 4.0, "p90": 6.0 }, "actions": { "n": 195, "mean": 4.62051282051282, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 195, "mean": 4.61025641025641, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 195, "mean": 240.90741776741774, "median": 223.5, "p90": 325.65 }, "generated_tokens": { "n": 195, "mean": 1130.8615384615384, "median": 996.0, "p90": 1494.0 }, "first_turn_tokens": { "n": 195, "mean": 190.22051282051282, "median": 191.0, "p90": 248.0 }, "searches": { "n": 195, "mean": 1.0615384615384615, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 195, "mean": 1.0615384615384615, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 195, "mean": 1.0564102564102564, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 195, "mean": 1.0461538461538462, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 195, "mean": 0.010256410256410256, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 195, "mean": 0.06153846153846154, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 195, "mean": 0.010256410256410256, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 195, "mean": 1.4205128205128206, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 195, "mean": 0.06666666666666667, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 195, "mean": 0.010256410256410256, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 195, "mean": 0.010256410256410256, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 195, "mean": 312.86686517186513, "median": 292.2, "p90": 426.0 }, "first_product_rank": { "n": 195, "mean": 1.5897435897435896, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 244.2059800664452, "token_complete_episodes": 195, "rates": { "protocol_errors": 0.010256410256410256, "product_revisits": 0.010256410256410256, "option_changes": 0.03076923076923077, "backtracks": 0.05641025641025641 }, "terminations": { "purchase": 195 } }, "failure": { "n": 61, "tasks": 13, "metrics": { "model_steps": { "n": 61, "mean": 6.639344262295082, "median": 5.0, "p90": 12.0 }, "actions": { "n": 61, "mean": 6.508196721311475, "median": 5.0, "p90": 12.0 }, "valid_actions": { "n": 61, "mean": 6.229508196721311, "median": 5.0, "p90": 10.0 }, "tokens_per_turn": { "n": 61, "mean": 275.49848968132477, "median": 269.29411764705884, "p90": 337.0 }, "generated_tokens": { "n": 61, "mean": 1758.5901639344263, "median": 1241.0, "p90": 3703.0 }, "first_turn_tokens": { "n": 61, "mean": 202.5737704918033, "median": 204.0, "p90": 261.0 }, "searches": { "n": 61, "mean": 1.540983606557377, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 61, "mean": 1.5245901639344261, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 61, "mean": 1.5081967213114753, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 61, "mean": 1.3278688524590163, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 61, "mean": 0.18032786885245902, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 61, "mean": 0.5573770491803278, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 61, "mean": 0.13114754098360656, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 61, "mean": 1.540983606557377, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 61, "mean": 0.11475409836065574, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 61, "mean": 0.08196721311475409, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 61, "mean": 0.2786885245901639, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 61, "mean": 365.15270487682255, "median": 361.0, "p90": 469.5 }, "first_product_rank": { "n": 61, "mean": 3.9672131147540983, "median": 1.0, "p90": 8.0 } }, "success_rate": 0.0, "reward_mean": 0.04918032786885246, "pooled_tokens_per_turn": 264.8740740740741, "token_complete_episodes": 61, "rates": { "protocol_errors": 0.03278688524590164, "product_revisits": 0.11475409836065574, "option_changes": 0.03278688524590164, "backtracks": 0.2786885245901639 }, "terminations": { "purchase": 58, "generation_length": 3 } } }, { "step": 35, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.49609375, "median": 4.0, "p90": 8.0 }, "actions": { "n": 256, "mean": 5.39453125, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 4.95703125, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 248.67280621256188, "median": 239.25, "p90": 315.75 }, "generated_tokens": { "n": 256, "mean": 1344.33984375, "median": 1028.0, "p90": 2379.5 }, "first_turn_tokens": { "n": 256, "mean": 199.27734375, "median": 201.5, "p90": 254.5 }, "searches": { "n": 256, "mean": 1.25, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.24609375, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.21875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.09375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.25390625, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.2890625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.4375, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 322.79719221802065, "median": 312.75, "p90": 418.2 }, "first_product_rank": { "n": 256, "mean": 1.45703125, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.875, "reward_mean": 0.875, "pooled_tokens_per_turn": 244.59914712153517, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.03515625, "product_revisits": 0.078125, "option_changes": 0.01953125, "backtracks": 0.12109375 }, "terminations": { "purchase": 242, "generation_length": 14 } }, "success": { "n": 224, "tasks": 29, "metrics": { "model_steps": { "n": 224, "mean": 4.392857142857143, "median": 4.0, "p90": 5.0 }, "actions": { "n": 224, "mean": 4.379464285714286, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 224, "mean": 4.379464285714286, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 224, "mean": 246.7737492913832, "median": 238.625, "p90": 308.20000000000005 }, "generated_tokens": { "n": 224, "mean": 1083.0848214285713, "median": 1003.0, "p90": 1373.6000000000004 }, "first_turn_tokens": { "n": 224, "mean": 199.34375, "median": 201.5, "p90": 257.0 }, "searches": { "n": 224, "mean": 1.0446428571428572, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 224, "mean": 1.0401785714285714, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 224, "mean": 1.0446428571428572, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 224, "mean": 1.03125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 224, "mean": 0.013392857142857142, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 224, "mean": 0.044642857142857144, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 224, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 224, "mean": 1.2455357142857142, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 224, "mean": 0.008928571428571428, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 224, "mean": 0.013392857142857142, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 224, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 224, "mean": 321.19995216836736, "median": 311.55, "p90": 410.65000000000003 }, "first_product_rank": { "n": 224, "mean": 1.4464285714285714, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 246.5558943089431, "token_complete_episodes": 224, "rates": { "protocol_errors": 0.013392857142857142, "product_revisits": 0.008928571428571428, "option_changes": 0.008928571428571428, "backtracks": 0.04017857142857143 }, "terminations": { "purchase": 224 } }, "failure": { "n": 32, "tasks": 6, "metrics": { "model_steps": { "n": 32, "mean": 13.21875, "median": 14.5, "p90": 21.0 }, "actions": { "n": 32, "mean": 12.5, "median": 14.5, "p90": 20.0 }, "valid_actions": { "n": 32, "mean": 9.0, "median": 10.0, "p90": 13.0 }, "tokens_per_turn": { "n": 32, "mean": 261.9662046608124, "median": 251.79166666666669, "p90": 335.225 }, "generated_tokens": { "n": 32, "mean": 3173.125, "median": 3095.0, "p90": 4963.8 }, "first_turn_tokens": { "n": 32, "mean": 198.8125, "median": 204.0, "p90": 248.60000000000002 }, "searches": { "n": 32, "mean": 2.6875, "median": 3.0, "p90": 4.0 }, "unique_queries": { "n": 32, "mean": 2.6875, "median": 3.0, "p90": 4.0 }, "product_opens": { "n": 32, "mean": 2.4375, "median": 3.0, "p90": 4.0 }, "unique_products": { "n": 32, "mean": 1.53125, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 32, "mean": 0.90625, "median": 1.0, "p90": 2.0 }, "backtracks": { "n": 32, "mean": 1.71875, "median": 2.0, "p90": 3.0 }, "pagination": { "n": 32, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 32, "mean": 1.59375, "median": 1.0, "p90": 3.900000000000002 }, "option_changes": { "n": 32, "mean": 0.3125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 32, "mean": 0.28125, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 32, "mean": 3.5, "median": 2.5, "p90": 8.900000000000002 }, "reasoning_chars_per_turn": { "n": 32, "mean": 333.9778725655939, "median": 325.38690476190476, "p90": 438.85 }, "first_product_rank": { "n": 32, "mean": 1.53125, "median": 1.0, "p90": 3.900000000000002 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 240.04728132387706, "token_complete_episodes": 32, "rates": { "protocol_errors": 0.1875, "product_revisits": 0.5625, "option_changes": 0.09375, "backtracks": 0.6875 }, "terminations": { "purchase": 18, "generation_length": 14 } } }, { "step": 36, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.6328125, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.60546875, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.59375, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 239.9293064471621, "median": 237.0, "p90": 298.5 }, "generated_tokens": { "n": 256, "mean": 1127.328125, "median": 1001.5, "p90": 1435.0 }, "first_turn_tokens": { "n": 256, "mean": 177.84765625, "median": 176.0, "p90": 233.5 }, "searches": { "n": 256, "mean": 1.1015625, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.1015625, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.08984375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.06640625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.1015625, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.30078125, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 310.7324785737152, "median": 307.875, "p90": 398.25 }, "first_product_rank": { "n": 256, "mean": 1.5234375, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.88671875, "reward_mean": 0.8919270833333334, "pooled_tokens_per_turn": 243.33558178752108, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.015625, "product_revisits": 0.015625, "option_changes": 0.01953125, "backtracks": 0.0703125 }, "terminations": { "purchase": 254, "generation_length": 2 } }, "success": { "n": 227, "tasks": 31, "metrics": { "model_steps": { "n": 227, "mean": 4.378854625550661, "median": 4.0, "p90": 5.0 }, "actions": { "n": 227, "mean": 4.370044052863436, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 227, "mean": 4.361233480176211, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 227, "mean": 233.62595527331212, "median": 228.5, "p90": 292.58 }, "generated_tokens": { "n": 227, "mean": 1024.920704845815, "median": 969.0, "p90": 1314.8000000000002 }, "first_turn_tokens": { "n": 227, "mean": 175.1057268722467, "median": 171.0, "p90": 232.60000000000005 }, "searches": { "n": 227, "mean": 1.0352422907488987, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 227, "mean": 1.0352422907488987, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 227, "mean": 1.0352422907488987, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 227, "mean": 1.026431718061674, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 227, "mean": 0.00881057268722467, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 227, "mean": 0.03524229074889868, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 227, "mean": 0.00881057268722467, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 227, "mean": 1.2466960352422907, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 227, "mean": 0.030837004405286344, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 227, "mean": 0.00881057268722467, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 227, "mean": 0.00881057268722467, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 227, "mean": 302.6758176478441, "median": 296.75, "p90": 386.1571428571429 }, "first_product_rank": { "n": 227, "mean": 1.2819383259911894, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 234.06136820925553, "token_complete_episodes": 227, "rates": { "protocol_errors": 0.00881057268722467, "product_revisits": 0.00881057268722467, "option_changes": 0.01762114537444934, "backtracks": 0.03524229074889868 }, "terminations": { "purchase": 227 } }, "failure": { "n": 29, "tasks": 10, "metrics": { "model_steps": { "n": 29, "mean": 6.620689655172414, "median": 5.0, "p90": 8.999999999999996 }, "actions": { "n": 29, "mean": 6.448275862068965, "median": 5.0, "p90": 8.799999999999997 }, "valid_actions": { "n": 29, "mean": 6.413793103448276, "median": 5.0, "p90": 8.799999999999997 }, "tokens_per_turn": { "n": 29, "mean": 289.2693311528157, "median": 280.875, "p90": 332.25 }, "generated_tokens": { "n": 29, "mean": 1928.9310344827586, "median": 1323.0, "p90": 3304.7999999999997 }, "first_turn_tokens": { "n": 29, "mean": 199.31034482758622, "median": 201.0, "p90": 230.0 }, "searches": { "n": 29, "mean": 1.6206896551724137, "median": 1.0, "p90": 2.1999999999999993 }, "unique_queries": { "n": 29, "mean": 1.6206896551724137, "median": 1.0, "p90": 2.1999999999999993 }, "product_opens": { "n": 29, "mean": 1.5172413793103448, "median": 1.0, "p90": 2.1999999999999993 }, "unique_products": { "n": 29, "mean": 1.3793103448275863, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 29, "mean": 0.13793103448275862, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 29, "mean": 0.6206896551724138, "median": 0.0, "p90": 1.1999999999999993 }, "pagination": { "n": 29, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 29, "mean": 1.7241379310344827, "median": 2.0, "p90": 2.1999999999999993 }, "option_changes": { "n": 29, "mean": 0.034482758620689655, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 29, "mean": 0.10344827586206896, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 29, "mean": 0.034482758620689655, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 29, "mean": 373.7966865107065, "median": 365.875, "p90": 446.8345238095238 }, "first_product_rank": { "n": 29, "mean": 3.413793103448276, "median": 2.0, "p90": 8.2 } }, "success_rate": 0.0, "reward_mean": 0.04597701149425287, "pooled_tokens_per_turn": 291.3489583333333, "token_complete_episodes": 29, "rates": { "protocol_errors": 0.06896551724137931, "product_revisits": 0.06896551724137931, "option_changes": 0.034482758620689655, "backtracks": 0.3448275862068966 }, "terminations": { "purchase": 27, "generation_length": 2 } } }, { "step": 37, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.9609375, "median": 4.0, "p90": 6.0 }, "actions": { "n": 256, "mean": 4.9296875, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 256, "mean": 4.83984375, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 256, "mean": 246.9541020466843, "median": 241.5, "p90": 322.4307692307692 }, "generated_tokens": { "n": 256, "mean": 1240.99609375, "median": 1042.0, "p90": 1727.5 }, "first_turn_tokens": { "n": 256, "mean": 180.21484375, "median": 177.5, "p90": 234.5 }, "searches": { "n": 256, "mean": 1.14453125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.13671875, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.140625, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.08203125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.05859375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.14453125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.41796875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.08984375, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 316.2586170291831, "median": 308.0, "p90": 427.8 }, "first_product_rank": { "n": 256, "mean": 1.65234375, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.796875, "reward_mean": 0.8015625, "pooled_tokens_per_turn": 250.1535433070866, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.01171875, "product_revisits": 0.03515625, "option_changes": 0.0234375, "backtracks": 0.08984375 }, "terminations": { "purchase": 253, "generation_length": 3 } }, "success": { "n": 204, "tasks": 28, "metrics": { "model_steps": { "n": 204, "mean": 4.485294117647059, "median": 4.0, "p90": 5.0 }, "actions": { "n": 204, "mean": 4.480392156862745, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 204, "mean": 4.455882352941177, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 204, "mean": 239.5316291687615, "median": 234.625, "p90": 314.02500000000003 }, "generated_tokens": { "n": 204, "mean": 1080.2107843137255, "median": 1008.0, "p90": 1401.1000000000004 }, "first_turn_tokens": { "n": 204, "mean": 181.5343137254902, "median": 180.5, "p90": 237.70000000000002 }, "searches": { "n": 204, "mean": 1.0441176470588236, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 204, "mean": 1.0392156862745099, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 204, "mean": 1.0441176470588236, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 204, "mean": 1.0343137254901962, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 204, "mean": 0.00980392156862745, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 204, "mean": 0.04411764705882353, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 204, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 204, "mean": 1.3235294117647058, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 204, "mean": 0.024509803921568627, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 204, "mean": 0.004901960784313725, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 204, "mean": 0.024509803921568627, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 204, "mean": 305.5052758924083, "median": 298.125, "p90": 411.67500000000007 }, "first_product_rank": { "n": 204, "mean": 1.3529411764705883, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 240.83387978142076, "token_complete_episodes": 204, "rates": { "protocol_errors": 0.004901960784313725, "product_revisits": 0.00980392156862745, "option_changes": 0.0196078431372549, "backtracks": 0.0392156862745098 }, "terminations": { "purchase": 204 } }, "failure": { "n": 52, "tasks": 13, "metrics": { "model_steps": { "n": 52, "mean": 6.826923076923077, "median": 4.5, "p90": 13.899999999999999 }, "actions": { "n": 52, "mean": 6.6923076923076925, "median": 4.5, "p90": 13.899999999999999 }, "valid_actions": { "n": 52, "mean": 6.346153846153846, "median": 4.0, "p90": 12.899999999999999 }, "tokens_per_turn": { "n": 52, "mean": 276.07303410622757, "median": 273.34375, "p90": 348.025 }, "generated_tokens": { "n": 52, "mean": 1871.7692307692307, "median": 1165.0, "p90": 4070.8999999999996 }, "first_turn_tokens": { "n": 52, "mean": 175.03846153846155, "median": 171.0, "p90": 214.0 }, "searches": { "n": 52, "mean": 1.5384615384615385, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 52, "mean": 1.5192307692307692, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 52, "mean": 1.5192307692307692, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 52, "mean": 1.2692307692307692, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 52, "mean": 0.25, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 52, "mean": 0.5384615384615384, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 52, "mean": 0.019230769230769232, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 52, "mean": 1.7884615384615385, "median": 1.0, "p90": 3.8999999999999986 }, "option_changes": { "n": 52, "mean": 0.1346153846153846, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 52, "mean": 0.07692307692307693, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 52, "mean": 0.34615384615384615, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 52, "mean": 358.4448014888381, "median": 349.75, "p90": 463.39000000000004 }, "first_product_rank": { "n": 52, "mean": 2.826923076923077, "median": 1.0, "p90": 9.0 } }, "success_rate": 0.0, "reward_mean": 0.023076923076923075, "pooled_tokens_per_turn": 274.17464788732394, "token_complete_episodes": 52, "rates": { "protocol_errors": 0.038461538461538464, "product_revisits": 0.1346153846153846, "option_changes": 0.038461538461538464, "backtracks": 0.28846153846153844 }, "terminations": { "purchase": 49, "generation_length": 3 } } }, { "step": 38, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.57421875, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.55859375, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.546875, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 220.23306699810607, "median": 212.0, "p90": 281.65 }, "generated_tokens": { "n": 256, "mean": 1009.65625, "median": 900.5, "p90": 1324.0 }, "first_turn_tokens": { "n": 256, "mean": 165.1953125, "median": 162.5, "p90": 212.5 }, "searches": { "n": 256, "mean": 1.08203125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.08203125, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.078125, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.03125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.08203125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.3046875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 280.8014221455628, "median": 268.125, "p90": 371.125 }, "first_product_rank": { "n": 256, "mean": 1.76953125, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.828125, "reward_mean": 0.84375, "pooled_tokens_per_turn": 220.72758326216908, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.00390625, "product_revisits": 0.0234375, "option_changes": 0.046875, "backtracks": 0.05078125 }, "terminations": { "purchase": 253, "generation_length": 3 } }, "success": { "n": 212, "tasks": 29, "metrics": { "model_steps": { "n": 212, "mean": 4.410377358490566, "median": 4.0, "p90": 5.0 }, "actions": { "n": 212, "mean": 4.410377358490566, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 212, "mean": 4.400943396226415, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 212, "mean": 213.73767084592555, "median": 207.7, "p90": 275.4 }, "generated_tokens": { "n": 212, "mean": 940.9811320754717, "median": 872.5, "p90": 1232.8000000000002 }, "first_turn_tokens": { "n": 212, "mean": 165.99528301886792, "median": 163.5, "p90": 213.9 }, "searches": { "n": 212, "mean": 1.0235849056603774, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 212, "mean": 1.0235849056603774, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 212, "mean": 1.0235849056603774, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 212, "mean": 1.0188679245283019, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 212, "mean": 0.0047169811320754715, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 212, "mean": 0.02358490566037736, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 212, "mean": 0.014150943396226415, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 212, "mean": 1.3160377358490567, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 212, "mean": 0.05660377358490566, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 212, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 212, "mean": 0.009433962264150943, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 212, "mean": 270.28583135941625, "median": 259.25, "p90": 357.675 }, "first_product_rank": { "n": 212, "mean": 1.5754716981132075, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 213.3561497326203, "token_complete_episodes": 212, "rates": { "protocol_errors": 0.0, "product_revisits": 0.0047169811320754715, "option_changes": 0.05188679245283019, "backtracks": 0.02358490566037736 }, "terminations": { "purchase": 212 } }, "failure": { "n": 44, "tasks": 9, "metrics": { "model_steps": { "n": 44, "mean": 5.363636363636363, "median": 4.0, "p90": 8.700000000000003 }, "actions": { "n": 44, "mean": 5.2727272727272725, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 44, "mean": 5.25, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 44, "mean": 251.5290666404303, "median": 236.07954545454544, "p90": 322.275 }, "generated_tokens": { "n": 44, "mean": 1340.5454545454545, "median": 1006.5, "p90": 2382.7000000000003 }, "first_turn_tokens": { "n": 44, "mean": 161.3409090909091, "median": 156.0, "p90": 193.7 }, "searches": { "n": 44, "mean": 1.3636363636363635, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 44, "mean": 1.3636363636363635, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 44, "mean": 1.3409090909090908, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 44, "mean": 1.0909090909090908, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 44, "mean": 0.25, "median": 0.0, "p90": 0.7000000000000028 }, "backtracks": { "n": 44, "mean": 0.36363636363636365, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 44, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 44, "mean": 1.25, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 44, "mean": 0.022727272727272728, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 44, "mean": 0.022727272727272728, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 44, "mean": 0.022727272727272728, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 44, "mean": 331.4674504788141, "median": 310.375, "p90": 462.35 }, "first_product_rank": { "n": 44, "mean": 2.7045454545454546, "median": 1.0, "p90": 7.0 } }, "success_rate": 0.0, "reward_mean": 0.09090909090909091, "pooled_tokens_per_turn": 249.9322033898305, "token_complete_episodes": 44, "rates": { "protocol_errors": 0.022727272727272728, "product_revisits": 0.11363636363636363, "option_changes": 0.022727272727272728, "backtracks": 0.18181818181818182 }, "terminations": { "purchase": 41, "generation_length": 3 } } }, { "step": 39, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.06640625, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.9921875, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.984375, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 213.78411625022108, "median": 206.67142857142858, "p90": 266.42261904761904 }, "generated_tokens": { "n": 256, "mean": 1089.06640625, "median": 912.0, "p90": 1416.0 }, "first_turn_tokens": { "n": 256, "mean": 160.68359375, "median": 158.0, "p90": 204.5 }, "searches": { "n": 256, "mean": 1.15234375, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.15234375, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.12890625, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.078125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.15625, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.5859375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.20703125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 268.6288601270899, "median": 256.24038461538464, "p90": 351.6 }, "first_product_rank": { "n": 256, "mean": 1.7890625, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.828125, "reward_mean": 0.845703125, "pooled_tokens_per_turn": 214.95836545875096, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0234375, "product_revisits": 0.046875, "option_changes": 0.0625, "backtracks": 0.0703125 }, "terminations": { "purchase": 246, "generation_length": 10 } }, "success": { "n": 212, "tasks": 29, "metrics": { "model_steps": { "n": 212, "mean": 4.452830188679245, "median": 4.0, "p90": 5.0 }, "actions": { "n": 212, "mean": 4.44811320754717, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 212, "mean": 4.44811320754717, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 212, "mean": 213.6554412306771, "median": 205.625, "p90": 263.15 }, "generated_tokens": { "n": 212, "mean": 954.4150943396227, "median": 899.0, "p90": 1248.0 }, "first_turn_tokens": { "n": 212, "mean": 160.48584905660377, "median": 160.0, "p90": 202.9 }, "searches": { "n": 212, "mean": 1.0377358490566038, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 212, "mean": 1.0377358490566038, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 212, "mean": 1.0330188679245282, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 212, "mean": 1.0235849056603774, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 212, "mean": 0.009433962264150943, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 212, "mean": 0.03773584905660377, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 212, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 212, "mean": 1.3396226415094339, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 212, "mean": 0.06132075471698113, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 212, "mean": 0.0047169811320754715, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 212, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 212, "mean": 268.0082650839726, "median": 256.0, "p90": 352.0400000000001 }, "first_product_rank": { "n": 212, "mean": 1.721698113207547, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 214.33898305084745, "token_complete_episodes": 212, "rates": { "protocol_errors": 0.0047169811320754715, "product_revisits": 0.009433962264150943, "option_changes": 0.0330188679245283, "backtracks": 0.02830188679245283 }, "terminations": { "purchase": 212 } }, "failure": { "n": 44, "tasks": 10, "metrics": { "model_steps": { "n": 44, "mean": 8.022727272727273, "median": 5.0, "p90": 19.0 }, "actions": { "n": 44, "mean": 7.613636363636363, "median": 5.0, "p90": 16.0 }, "valid_actions": { "n": 44, "mean": 7.568181818181818, "median": 5.0, "p90": 16.0 }, "tokens_per_turn": { "n": 44, "mean": 214.4040958898421, "median": 210.4, "p90": 270.085 }, "generated_tokens": { "n": 44, "mean": 1737.840909090909, "median": 1012.5, "p90": 4041.400000000001 }, "first_turn_tokens": { "n": 44, "mean": 161.63636363636363, "median": 151.0, "p90": 216.10000000000002 }, "searches": { "n": 44, "mean": 1.7045454545454546, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 44, "mean": 1.7045454545454546, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 44, "mean": 1.5909090909090908, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 44, "mean": 1.3409090909090908, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 44, "mean": 0.25, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 44, "mean": 0.7272727272727273, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 44, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 44, "mean": 2.772727272727273, "median": 2.0, "p90": 6.0 }, "option_changes": { "n": 44, "mean": 0.9090909090909091, "median": 0.0, "p90": 3.0 }, "protocol_errors": { "n": 44, "mean": 0.18181818181818182, "median": 0.0, "p90": 0.7000000000000028 }, "invalid_actions": { "n": 44, "mean": 0.045454545454545456, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 44, "mean": 271.6189998802914, "median": 270.9090909090909, "p90": 348.995 }, "first_product_rank": { "n": 44, "mean": 2.1136363636363638, "median": 1.0, "p90": 4.800000000000011 } }, "success_rate": 0.0, "reward_mean": 0.10227272727272728, "pooled_tokens_per_turn": 216.61473087818698, "token_complete_episodes": 44, "rates": { "protocol_errors": 0.11363636363636363, "product_revisits": 0.22727272727272727, "option_changes": 0.20454545454545456, "backtracks": 0.2727272727272727 }, "terminations": { "purchase": 34, "generation_length": 10 } } }, { "step": 40, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.1015625, "median": 4.0, "p90": 8.0 }, "actions": { "n": 256, "mean": 5.0546875, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.03515625, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 215.7193513173743, "median": 209.25, "p90": 277.125 }, "generated_tokens": { "n": 256, "mean": 1115.09375, "median": 876.0, "p90": 1777.5 }, "first_turn_tokens": { "n": 256, "mean": 157.484375, "median": 155.5, "p90": 204.5 }, "searches": { "n": 256, "mean": 1.203125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.1953125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.18359375, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.0703125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.11328125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.203125, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.44140625, "median": 1.0, "p90": 2.5 }, "option_changes": { "n": 256, "mean": 0.1171875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 274.07978573623257, "median": 263.16666666666663, "p90": 368.45 }, "first_product_rank": { "n": 256, "mean": 1.39453125, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.82421875, "reward_mean": 0.82734375, "pooled_tokens_per_turn": 218.57886676875958, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.01953125, "product_revisits": 0.078125, "option_changes": 0.08203125, "backtracks": 0.125 }, "terminations": { "purchase": 249, "generation_length": 7 } }, "success": { "n": 211, "tasks": 30, "metrics": { "model_steps": { "n": 211, "mean": 4.7725118483412325, "median": 4.0, "p90": 6.0 }, "actions": { "n": 211, "mean": 4.758293838862559, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 211, "mean": 4.7440758293838865, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 211, "mean": 210.55102179907618, "median": 206.5, "p90": 270.0 }, "generated_tokens": { "n": 211, "mean": 1014.6682464454976, "median": 868.0, "p90": 1423.0 }, "first_turn_tokens": { "n": 211, "mean": 157.92417061611374, "median": 156.0, "p90": 204.0 }, "searches": { "n": 211, "mean": 1.113744075829384, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 211, "mean": 1.113744075829384, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 211, "mean": 1.113744075829384, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 211, "mean": 1.04739336492891, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 211, "mean": 0.06635071090047394, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 211, "mean": 0.11374407582938388, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 211, "mean": 0.009478672985781991, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 211, "mean": 1.3933649289099526, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 211, "mean": 0.0947867298578199, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 211, "mean": 0.014218009478672985, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 211, "mean": 0.014218009478672985, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 211, "mean": 265.1153446502224, "median": 257.5, "p90": 351.44444444444446 }, "first_product_rank": { "n": 211, "mean": 1.2843601895734598, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 212.6067527308838, "token_complete_episodes": 211, "rates": { "protocol_errors": 0.014218009478672985, "product_revisits": 0.05687203791469194, "option_changes": 0.06635071090047394, "backtracks": 0.09004739336492891 }, "terminations": { "purchase": 211 } }, "failure": { "n": 45, "tasks": 12, "metrics": { "model_steps": { "n": 45, "mean": 6.644444444444445, "median": 4.0, "p90": 13.600000000000001 }, "actions": { "n": 45, "mean": 6.444444444444445, "median": 4.0, "p90": 12.600000000000001 }, "valid_actions": { "n": 45, "mean": 6.4, "median": 4.0, "p90": 12.600000000000001 }, "tokens_per_turn": { "n": 45, "mean": 239.95307416983889, "median": 246.8, "p90": 292.35 }, "generated_tokens": { "n": 45, "mean": 1585.9777777777779, "median": 1155.0, "p90": 3130.8 }, "first_turn_tokens": { "n": 45, "mean": 155.42222222222222, "median": 149.0, "p90": 204.8 }, "searches": { "n": 45, "mean": 1.6222222222222222, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 45, "mean": 1.5777777777777777, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 45, "mean": 1.511111111111111, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 45, "mean": 1.1777777777777778, "median": 1.0, "p90": 1.6000000000000014 }, "product_revisits": { "n": 45, "mean": 0.3333333333333333, "median": 0.0, "p90": 1.6000000000000014 }, "backtracks": { "n": 45, "mean": 0.6222222222222222, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 45, "mean": 0.13333333333333333, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 45, "mean": 1.6666666666666667, "median": 1.0, "p90": 4.0 }, "option_changes": { "n": 45, "mean": 0.2222222222222222, "median": 0.0, "p90": 1.0 }, "protocol_errors": { "n": 45, "mean": 0.044444444444444446, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 45, "mean": 0.044444444444444446, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 45, "mean": 316.1130539395245, "median": 332.75, "p90": 400.6 }, "first_product_rank": { "n": 45, "mean": 1.9111111111111112, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.0, "reward_mean": 0.017777777777777778, "pooled_tokens_per_turn": 238.69230769230768, "token_complete_episodes": 45, "rates": { "protocol_errors": 0.044444444444444446, "product_revisits": 0.17777777777777778, "option_changes": 0.15555555555555556, "backtracks": 0.28888888888888886 }, "terminations": { "purchase": 38, "generation_length": 7 } } }, { "step": 41, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.80859375, "median": 4.0, "p90": 7.0 }, "actions": { "n": 256, "mean": 5.73046875, "median": 4.0, "p90": 7.0 }, "valid_actions": { "n": 256, "mean": 5.2578125, "median": 4.0, "p90": 7.0 }, "tokens_per_turn": { "n": 256, "mean": 205.85355437710638, "median": 198.775, "p90": 256.875 }, "generated_tokens": { "n": 256, "mean": 1154.140625, "median": 896.5, "p90": 1640.5 }, "first_turn_tokens": { "n": 256, "mean": 158.55078125, "median": 145.0, "p90": 222.0 }, "searches": { "n": 256, "mean": 1.28515625, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.2578125, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.265625, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.06640625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.19921875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.30078125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.46875, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.47265625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 257.22883740595887, "median": 249.625, "p90": 325.875 }, "first_product_rank": { "n": 256, "mean": 1.58203125, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.92578125, "reward_mean": 0.92578125, "pooled_tokens_per_turn": 198.6953597848016, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.015625, "product_revisits": 0.07421875, "option_changes": 0.07421875, "backtracks": 0.09765625 }, "terminations": { "purchase": 240, "action_limit": 1, "generation_length": 15 } }, "success": { "n": 237, "tasks": 30, "metrics": { "model_steps": { "n": 237, "mean": 4.523206751054852, "median": 4.0, "p90": 5.0 }, "actions": { "n": 237, "mean": 4.523206751054852, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 237, "mean": 4.518987341772152, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 237, "mean": 207.54386871918516, "median": 201.0, "p90": 258.0 }, "generated_tokens": { "n": 237, "mean": 934.5991561181435, "median": 870.0, "p90": 1232.8 }, "first_turn_tokens": { "n": 237, "mean": 158.62869198312237, "median": 145.0, "p90": 224.4 }, "searches": { "n": 237, "mean": 1.0421940928270041, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 237, "mean": 1.0421940928270041, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 237, "mean": 1.0421940928270041, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 237, "mean": 1.0337552742616034, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 237, "mean": 0.008438818565400843, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 237, "mean": 0.04219409282700422, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 237, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 237, "mean": 1.3924050632911393, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 237, "mean": 0.08438818565400844, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 237, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 237, "mean": 0.004219409282700422, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 237, "mean": 259.4846269763991, "median": 251.75, "p90": 326.7 }, "first_product_rank": { "n": 237, "mean": 1.5949367088607596, "median": 1.0, "p90": 3.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 206.62313432835822, "token_complete_episodes": 237, "rates": { "protocol_errors": 0.0, "product_revisits": 0.008438818565400843, "option_changes": 0.08016877637130802, "backtracks": 0.03375527426160337 }, "terminations": { "purchase": 237 } }, "failure": { "n": 19, "tasks": 4, "metrics": { "model_steps": { "n": 19, "mean": 21.842105263157894, "median": 22.0, "p90": 29.2 }, "actions": { "n": 19, "mean": 20.789473684210527, "median": 21.0, "p90": 28.2 }, "valid_actions": { "n": 19, "mean": 14.473684210526315, "median": 14.0, "p90": 19.2 }, "tokens_per_turn": { "n": 19, "mean": 184.7691070574918, "median": 188.5, "p90": 206.74212454212454 }, "generated_tokens": { "n": 19, "mean": 3892.6315789473683, "median": 4235.0, "p90": 5016.8 }, "first_turn_tokens": { "n": 19, "mean": 157.57894736842104, "median": 145.0, "p90": 204.6 }, "searches": { "n": 19, "mean": 4.315789473684211, "median": 5.0, "p90": 5.0 }, "unique_queries": { "n": 19, "mean": 3.9473684210526314, "median": 4.0, "p90": 5.0 }, "product_opens": { "n": 19, "mean": 4.052631578947368, "median": 4.0, "p90": 5.0 }, "unique_products": { "n": 19, "mean": 1.4736842105263157, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 19, "mean": 2.5789473684210527, "median": 3.0, "p90": 4.0 }, "backtracks": { "n": 19, "mean": 3.526315789473684, "median": 4.0, "p90": 5.0 }, "pagination": { "n": 19, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 19, "mean": 2.4210526315789473, "median": 3.0, "p90": 5.0 }, "option_changes": { "n": 19, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 19, "mean": 0.2631578947368421, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 19, "mean": 6.315789473684211, "median": 6.0, "p90": 10.0 }, "reasoning_chars_per_turn": { "n": 19, "mean": 229.09083065888876, "median": 231.8, "p90": 263.41611721611724 }, "first_product_rank": { "n": 19, "mean": 1.4210526315789473, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 178.21686746987953, "token_complete_episodes": 19, "rates": { "protocol_errors": 0.21052631578947367, "product_revisits": 0.8947368421052632, "option_changes": 0.0, "backtracks": 0.8947368421052632 }, "terminations": { "purchase": 3, "action_limit": 1, "generation_length": 15 } } }, { "step": 42, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.21875, "median": 4.0, "p90": 12.0 }, "actions": { "n": 256, "mean": 6.08984375, "median": 4.0, "p90": 11.5 }, "valid_actions": { "n": 256, "mean": 5.53125, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 256, "mean": 214.86219744764952, "median": 204.5, "p90": 271.25 }, "generated_tokens": { "n": 256, "mean": 1365.76171875, "median": 908.5, "p90": 2593.0 }, "first_turn_tokens": { "n": 256, "mean": 144.44140625, "median": 136.0, "p90": 202.0 }, "searches": { "n": 256, "mean": 1.33203125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.328125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.30078125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.0546875, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.24609375, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.3359375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.65234375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.28515625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.55859375, "median": 0.0, "p90": 2.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 268.42742504584135, "median": 257.3125, "p90": 356.93933823529414 }, "first_product_rank": { "n": 256, "mean": 1.73828125, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.75, "reward_mean": 0.76171875, "pooled_tokens_per_turn": 219.61997487437185, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.02734375, "product_revisits": 0.12890625, "option_changes": 0.08203125, "backtracks": 0.140625 }, "terminations": { "purchase": 231, "generation_length": 25 } }, "success": { "n": 192, "tasks": 27, "metrics": { "model_steps": { "n": 192, "mean": 4.572916666666667, "median": 4.0, "p90": 5.0 }, "actions": { "n": 192, "mean": 4.567708333333333, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 192, "mean": 4.567708333333333, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 192, "mean": 201.69095906391405, "median": 194.5, "p90": 257.68 }, "generated_tokens": { "n": 192, "mean": 923.5104166666666, "median": 822.5, "p90": 1275.2 }, "first_turn_tokens": { "n": 192, "mean": 148.66145833333334, "median": 136.0, "p90": 205.0 }, "searches": { "n": 192, "mean": 1.0260416666666667, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 192, "mean": 1.0260416666666667, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 192, "mean": 1.0260416666666667, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 192, "mean": 1.0052083333333333, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 192, "mean": 0.020833333333333332, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 192, "mean": 0.026041666666666668, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 192, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 192, "mean": 1.4895833333333333, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 192, "mean": 0.20833333333333334, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 192, "mean": 0.005208333333333333, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 192, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 192, "mean": 253.9587547919809, "median": 242.0, "p90": 340.31 }, "first_product_rank": { "n": 192, "mean": 1.7708333333333333, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 201.9521640091116, "token_complete_episodes": 192, "rates": { "protocol_errors": 0.005208333333333333, "product_revisits": 0.020833333333333332, "option_changes": 0.0625, "backtracks": 0.020833333333333332 }, "terminations": { "purchase": 192 } }, "failure": { "n": 64, "tasks": 13, "metrics": { "model_steps": { "n": 64, "mean": 11.15625, "median": 10.0, "p90": 26.0 }, "actions": { "n": 64, "mean": 10.65625, "median": 9.5, "p90": 24.0 }, "valid_actions": { "n": 64, "mean": 8.421875, "median": 7.0, "p90": 17.700000000000003 }, "tokens_per_turn": { "n": 64, "mean": 254.37591259885608, "median": 232.99650349650352, "p90": 301.21875000000006 }, "generated_tokens": { "n": 64, "mean": 2692.515625, "median": 2222.5, "p90": 5127.1 }, "first_turn_tokens": { "n": 64, "mean": 131.78125, "median": 126.0, "p90": 179.90000000000003 }, "searches": { "n": 64, "mean": 2.25, "median": 1.5, "p90": 5.0 }, "unique_queries": { "n": 64, "mean": 2.234375, "median": 1.5, "p90": 5.0 }, "product_opens": { "n": 64, "mean": 2.125, "median": 1.0, "p90": 4.700000000000003 }, "unique_products": { "n": 64, "mean": 1.203125, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 64, "mean": 0.921875, "median": 0.0, "p90": 3.0 }, "backtracks": { "n": 64, "mean": 1.265625, "median": 0.5, "p90": 4.0 }, "pagination": { "n": 64, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 64, "mean": 2.140625, "median": 1.0, "p90": 5.0 }, "option_changes": { "n": 64, "mean": 0.515625, "median": 0.0, "p90": 1.7000000000000028 }, "protocol_errors": { "n": 64, "mean": 0.109375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 64, "mean": 2.234375, "median": 0.0, "p90": 7.0 }, "reasoning_chars_per_turn": { "n": 64, "mean": 311.8334358074228, "median": 312.4861111111111, "p90": 376.725 }, "first_product_rank": { "n": 64, "mean": 1.640625, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.0, "reward_mean": 0.046875, "pooled_tokens_per_turn": 241.34593837535013, "token_complete_episodes": 64, "rates": { "protocol_errors": 0.09375, "product_revisits": 0.453125, "option_changes": 0.140625, "backtracks": 0.5 }, "terminations": { "generation_length": 25, "purchase": 39 } } }, { "step": 43, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.71484375, "median": 4.0, "p90": 10.5 }, "actions": { "n": 256, "mean": 5.65625, "median": 4.0, "p90": 10.5 }, "valid_actions": { "n": 256, "mean": 5.63671875, "median": 4.0, "p90": 10.5 }, "tokens_per_turn": { "n": 256, "mean": 213.5624614762139, "median": 204.875, "p90": 278.1470588235294 }, "generated_tokens": { "n": 256, "mean": 1267.30078125, "median": 895.5, "p90": 2363.0 }, "first_turn_tokens": { "n": 256, "mean": 141.359375, "median": 134.0, "p90": 191.0 }, "searches": { "n": 256, "mean": 1.34765625, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.3203125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.33203125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.19921875, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.1328125, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.34765625, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.6171875, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.1328125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 272.2524695890497, "median": 264.375, "p90": 367.725 }, "first_product_rank": { "n": 256, "mean": 1.68359375, "median": 1.0, "p90": 4.0 } }, "success_rate": 0.8203125, "reward_mean": 0.8296875, "pooled_tokens_per_turn": 221.755980861244, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.01953125, "product_revisits": 0.10546875, "option_changes": 0.0546875, "backtracks": 0.21875 }, "terminations": { "purchase": 249, "generation_length": 7 } }, "success": { "n": 210, "tasks": 29, "metrics": { "model_steps": { "n": 210, "mean": 5.004761904761905, "median": 4.0, "p90": 8.0 }, "actions": { "n": 210, "mean": 4.985714285714286, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 210, "mean": 4.9714285714285715, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 210, "mean": 201.18915945122987, "median": 193.875, "p90": 258.4 }, "generated_tokens": { "n": 210, "mean": 1023.9714285714285, "median": 843.5, "p90": 1766.9999999999998 }, "first_turn_tokens": { "n": 210, "mean": 138.06190476190477, "median": 134.0, "p90": 182.1 }, "searches": { "n": 210, "mean": 1.2095238095238094, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 210, "mean": 1.180952380952381, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 210, "mean": 1.2095238095238094, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 210, "mean": 1.1285714285714286, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 210, "mean": 0.08095238095238096, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 210, "mean": 0.20952380952380953, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 210, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 210, "mean": 1.3428571428571427, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 210, "mean": 0.023809523809523808, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 210, "mean": 0.01904761904761905, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 210, "mean": 0.014285714285714285, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 210, "mean": 254.83828529184953, "median": 242.375, "p90": 342.62499999999994 }, "first_product_rank": { "n": 210, "mean": 1.4190476190476191, "median": 1.0, "p90": 3.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 204.59942911512846, "token_complete_episodes": 210, "rates": { "protocol_errors": 0.009523809523809525, "product_revisits": 0.0761904761904762, "option_changes": 0.023809523809523808, "backtracks": 0.15714285714285714 }, "terminations": { "purchase": 210 } }, "failure": { "n": 46, "tasks": 11, "metrics": { "model_steps": { "n": 46, "mean": 8.956521739130435, "median": 8.0, "p90": 17.0 }, "actions": { "n": 46, "mean": 8.717391304347826, "median": 8.0, "p90": 17.0 }, "valid_actions": { "n": 46, "mean": 8.673913043478262, "median": 8.0, "p90": 16.5 }, "tokens_per_turn": { "n": 46, "mean": 270.0492750685324, "median": 256.625, "p90": 323.75 }, "generated_tokens": { "n": 46, "mean": 2378.1521739130435, "median": 1858.5, "p90": 4707.0 }, "first_turn_tokens": { "n": 46, "mean": 156.41304347826087, "median": 131.5, "p90": 249.5 }, "searches": { "n": 46, "mean": 1.9782608695652173, "median": 1.5, "p90": 4.0 }, "unique_queries": { "n": 46, "mean": 1.9565217391304348, "median": 1.5, "p90": 4.0 }, "product_opens": { "n": 46, "mean": 1.891304347826087, "median": 1.5, "p90": 4.0 }, "unique_products": { "n": 46, "mean": 1.5217391304347827, "median": 1.0, "p90": 2.5 }, "product_revisits": { "n": 46, "mean": 0.3695652173913043, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 46, "mean": 0.9782608695652174, "median": 0.5, "p90": 3.0 }, "pagination": { "n": 46, "mean": 0.10869565217391304, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 46, "mean": 2.869565217391304, "median": 2.0, "p90": 7.0 }, "option_changes": { "n": 46, "mean": 0.6304347826086957, "median": 0.0, "p90": 1.5 }, "protocol_errors": { "n": 46, "mean": 0.08695652173913043, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 46, "mean": 0.043478260869565216, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 46, "mean": 351.7520065980071, "median": 346.5, "p90": 440.0 }, "first_product_rank": { "n": 46, "mean": 2.891304347826087, "median": 2.0, "p90": 5.0 } }, "success_rate": 0.0, "reward_mean": 0.05217391304347826, "pooled_tokens_per_turn": 265.5218446601942, "token_complete_episodes": 46, "rates": { "protocol_errors": 0.06521739130434782, "product_revisits": 0.2391304347826087, "option_changes": 0.1956521739130435, "backtracks": 0.5 }, "terminations": { "purchase": 39, "generation_length": 7 } } }, { "step": 44, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.08203125, "median": 4.0, "p90": 8.0 }, "actions": { "n": 256, "mean": 5.03515625, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.0, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 204.09789441301572, "median": 191.75, "p90": 255.425 }, "generated_tokens": { "n": 256, "mean": 1043.49609375, "median": 815.0, "p90": 1721.5 }, "first_turn_tokens": { "n": 256, "mean": 141.71484375, "median": 137.0, "p90": 199.5 }, "searches": { "n": 256, "mean": 1.203125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.1953125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.19140625, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.12890625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.0625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.203125, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.41015625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 254.37020263807372, "median": 237.75, "p90": 347.5 }, "first_product_rank": { "n": 256, "mean": 1.5234375, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.81640625, "reward_mean": 0.818359375, "pooled_tokens_per_turn": 205.3305149884704, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.01953125, "product_revisits": 0.05859375, "option_changes": 0.015625, "backtracks": 0.125 }, "terminations": { "purchase": 250, "generation_length": 6 } }, "success": { "n": 209, "tasks": 30, "metrics": { "model_steps": { "n": 209, "mean": 4.507177033492823, "median": 4.0, "p90": 6.0 }, "actions": { "n": 209, "mean": 4.497607655502392, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 209, "mean": 4.464114832535885, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 209, "mean": 199.40283439314538, "median": 191.75, "p90": 255.32 }, "generated_tokens": { "n": 209, "mean": 908.9043062200957, "median": 806.0, "p90": 1267.6000000000001 }, "first_turn_tokens": { "n": 209, "mean": 140.62200956937798, "median": 136.0, "p90": 196.0 }, "searches": { "n": 209, "mean": 1.0574162679425838, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 209, "mean": 1.0478468899521531, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 209, "mean": 1.0574162679425838, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 209, "mean": 1.0239234449760766, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 209, "mean": 0.03349282296650718, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 209, "mean": 0.05741626794258373, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 209, "mean": 0.009569377990430622, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 209, "mean": 1.2822966507177034, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 209, "mean": 0.019138755980861243, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 209, "mean": 0.009569377990430622, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 209, "mean": 0.03349282296650718, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 209, "mean": 252.59723361433888, "median": 234.5, "p90": 347.5 }, "first_product_rank": { "n": 209, "mean": 1.4162679425837321, "median": 1.0, "p90": 3.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 201.65711252653927, "token_complete_episodes": 209, "rates": { "protocol_errors": 0.009569377990430622, "product_revisits": 0.03349282296650718, "option_changes": 0.009569377990430622, "backtracks": 0.05741626794258373 }, "terminations": { "purchase": 209 } }, "failure": { "n": 47, "tasks": 13, "metrics": { "model_steps": { "n": 47, "mean": 7.638297872340425, "median": 4.0, "p90": 15.0 }, "actions": { "n": 47, "mean": 7.425531914893617, "median": 4.0, "p90": 14.399999999999999 }, "valid_actions": { "n": 47, "mean": 7.382978723404255, "median": 4.0, "p90": 13.799999999999997 }, "tokens_per_turn": { "n": 47, "mean": 224.97592726733262, "median": 193.75, "p90": 252.45 }, "generated_tokens": { "n": 47, "mean": 1642.0, "median": 1038.0, "p90": 3741.3999999999987 }, "first_turn_tokens": { "n": 47, "mean": 146.5744680851064, "median": 139.0, "p90": 202.0 }, "searches": { "n": 47, "mean": 1.851063829787234, "median": 1.0, "p90": 3.3999999999999986 }, "unique_queries": { "n": 47, "mean": 1.851063829787234, "median": 1.0, "p90": 3.3999999999999986 }, "product_opens": { "n": 47, "mean": 1.7872340425531914, "median": 1.0, "p90": 3.3999999999999986 }, "unique_products": { "n": 47, "mean": 1.5957446808510638, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 47, "mean": 0.19148936170212766, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 47, "mean": 0.851063829787234, "median": 0.0, "p90": 2.3999999999999986 }, "pagination": { "n": 47, "mean": 0.0425531914893617, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 47, "mean": 1.9787234042553192, "median": 1.0, "p90": 4.0 }, "option_changes": { "n": 47, "mean": 0.0425531914893617, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 47, "mean": 0.0851063829787234, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 47, "mean": 0.0425531914893617, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 47, "mean": 262.254256381916, "median": 241.75, "p90": 340.7 }, "first_product_rank": { "n": 47, "mean": 2.0, "median": 1.0, "p90": 4.0 } }, "success_rate": 0.0, "reward_mean": 0.010638297872340425, "pooled_tokens_per_turn": 214.9693593314763, "token_complete_episodes": 47, "rates": { "protocol_errors": 0.06382978723404255, "product_revisits": 0.1702127659574468, "option_changes": 0.0425531914893617, "backtracks": 0.425531914893617 }, "terminations": { "generation_length": 6, "purchase": 41 } } }, { "step": 45, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.921875, "median": 4.0, "p90": 6.0 }, "actions": { "n": 256, "mean": 4.8828125, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 256, "mean": 4.87109375, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 256, "mean": 195.1524912630772, "median": 186.625, "p90": 260.125 }, "generated_tokens": { "n": 256, "mean": 969.26171875, "median": 789.0, "p90": 1593.5 }, "first_turn_tokens": { "n": 256, "mean": 141.09375, "median": 131.0, "p90": 197.5 }, "searches": { "n": 256, "mean": 1.16015625, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.1484375, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.13671875, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0703125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.06640625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.16015625, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0390625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.38671875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 245.6652667521623, "median": 230.875, "p90": 357.4375 }, "first_product_rank": { "n": 256, "mean": 1.390625, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.90625, "reward_mean": 0.91748046875, "pooled_tokens_per_turn": 196.92936507936508, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0234375, "product_revisits": 0.05859375, "option_changes": 0.015625, "backtracks": 0.08984375 }, "terminations": { "purchase": 253, "generation_length": 3 } }, "success": { "n": 232, "tasks": 30, "metrics": { "model_steps": { "n": 232, "mean": 4.443965517241379, "median": 4.0, "p90": 5.0 }, "actions": { "n": 232, "mean": 4.435344827586207, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 232, "mean": 4.431034482758621, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 232, "mean": 192.63783524904215, "median": 184.125, "p90": 256.175 }, "generated_tokens": { "n": 232, "mean": 864.2629310344828, "median": 760.5, "p90": 1196.9 }, "first_turn_tokens": { "n": 232, "mean": 141.29741379310346, "median": 131.0, "p90": 195.9 }, "searches": { "n": 232, "mean": 1.0646551724137931, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 232, "mean": 1.0560344827586208, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 232, "mean": 1.0646551724137931, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 232, "mean": 1.0172413793103448, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 232, "mean": 0.04741379310344827, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 232, "mean": 0.06465517241379311, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 232, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 232, "mean": 1.2370689655172413, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 232, "mean": 0.017241379310344827, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 232, "mean": 0.008620689655172414, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 232, "mean": 0.004310344827586207, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 232, "mean": 239.79942870826494, "median": 227.625, "p90": 335.80000000000007 }, "first_product_rank": { "n": 232, "mean": 1.1896551724137931, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 194.48011639185256, "token_complete_episodes": 232, "rates": { "protocol_errors": 0.008620689655172414, "product_revisits": 0.04741379310344827, "option_changes": 0.01293103448275862, "backtracks": 0.05603448275862069 }, "terminations": { "purchase": 232 } }, "failure": { "n": 24, "tasks": 6, "metrics": { "model_steps": { "n": 24, "mean": 9.541666666666666, "median": 5.5, "p90": 18.7 }, "actions": { "n": 24, "mean": 9.208333333333334, "median": 5.5, "p90": 17.7 }, "valid_actions": { "n": 24, "mean": 9.125, "median": 5.5, "p90": 17.7 }, "tokens_per_turn": { "n": 24, "mean": 219.46083273208274, "median": 215.34761904761905, "p90": 269.27500000000003 }, "generated_tokens": { "n": 24, "mean": 1984.25, "median": 1573.0, "p90": 3837.7 }, "first_turn_tokens": { "n": 24, "mean": 139.125, "median": 129.5, "p90": 196.49999999999997 }, "searches": { "n": 24, "mean": 2.0833333333333335, "median": 1.0, "p90": 4.699999999999999 }, "unique_queries": { "n": 24, "mean": 2.0416666666666665, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 24, "mean": 1.8333333333333333, "median": 1.0, "p90": 3.6999999999999993 }, "unique_products": { "n": 24, "mean": 1.5833333333333333, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 24, "mean": 0.25, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 24, "mean": 1.0833333333333333, "median": 0.0, "p90": 3.6999999999999993 }, "pagination": { "n": 24, "mean": 0.4166666666666667, "median": 0.0, "p90": 1.6999999999999993 }, "option_selections": { "n": 24, "mean": 2.8333333333333335, "median": 2.0, "p90": 6.0 }, "option_changes": { "n": 24, "mean": 0.041666666666666664, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 24, "mean": 0.20833333333333334, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 24, "mean": 0.08333333333333333, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 24, "mean": 302.36836784317046, "median": 297.1166666666667, "p90": 388.175 }, "first_product_rank": { "n": 24, "mean": 3.3333333333333335, "median": 2.0, "p90": 8.299999999999994 } }, "success_rate": 0.0, "reward_mean": 0.11979166666666667, "pooled_tokens_per_turn": 207.95633187772927, "token_complete_episodes": 24, "rates": { "protocol_errors": 0.16666666666666666, "product_revisits": 0.16666666666666666, "option_changes": 0.041666666666666664, "backtracks": 0.4166666666666667 }, "terminations": { "purchase": 21, "generation_length": 3 } } }, { "step": 46, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.734375, "median": 4.0, "p90": 13.0 }, "actions": { "n": 256, "mean": 6.625, "median": 4.0, "p90": 13.0 }, "valid_actions": { "n": 256, "mean": 6.25, "median": 4.0, "p90": 12.0 }, "tokens_per_turn": { "n": 256, "mean": 189.8382597952667, "median": 180.72115384615384, "p90": 244.60576923076923 }, "generated_tokens": { "n": 256, "mean": 1309.03515625, "median": 848.0, "p90": 2687.0 }, "first_turn_tokens": { "n": 256, "mean": 131.90625, "median": 125.0, "p90": 178.0 }, "searches": { "n": 256, "mean": 1.4765625, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 256, "mean": 1.4609375, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 256, "mean": 1.41015625, "median": 1.0, "p90": 2.5 }, "unique_products": { "n": 256, "mean": 1.15234375, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.2578125, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.4765625, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 256, "mean": 0.06640625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.86328125, "median": 1.0, "p90": 4.0 }, "option_changes": { "n": 256, "mean": 0.25390625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.06640625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.375, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 238.49773304234014, "median": 225.64574898785426, "p90": 324.02777777777777 }, "first_product_rank": { "n": 256, "mean": 1.2109375, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.81640625, "reward_mean": 0.8232421875, "pooled_tokens_per_turn": 194.381090487239, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.046875, "product_revisits": 0.19140625, "option_changes": 0.09765625, "backtracks": 0.28125 }, "terminations": { "purchase": 245, "generation_length": 11 } }, "success": { "n": 209, "tasks": 29, "metrics": { "model_steps": { "n": 209, "mean": 5.717703349282297, "median": 4.0, "p90": 10.0 }, "actions": { "n": 209, "mean": 5.684210526315789, "median": 4.0, "p90": 9.200000000000017 }, "valid_actions": { "n": 209, "mean": 5.5311004784688995, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 209, "mean": 183.14430492655228, "median": 175.5, "p90": 235.28307692307695 }, "generated_tokens": { "n": 209, "mean": 1083.8612440191387, "median": 761.0, "p90": 1957.8000000000004 }, "first_turn_tokens": { "n": 209, "mean": 132.76076555023923, "median": 128.0, "p90": 178.0 }, "searches": { "n": 209, "mean": 1.277511961722488, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 209, "mean": 1.263157894736842, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 209, "mean": 1.277511961722488, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 209, "mean": 1.1100478468899522, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 209, "mean": 0.1674641148325359, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 209, "mean": 0.27751196172248804, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 209, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 209, "mean": 1.6985645933014355, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 209, "mean": 0.23444976076555024, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 209, "mean": 0.03349282296650718, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 209, "mean": 0.15311004784688995, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 209, "mean": 228.56064521915212, "median": 215.5, "p90": 311.20000000000005 }, "first_product_rank": { "n": 209, "mean": 1.2200956937799043, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 189.56234309623431, "token_complete_episodes": 209, "rates": { "protocol_errors": 0.023923444976076555, "product_revisits": 0.1339712918660287, "option_changes": 0.09569377990430622, "backtracks": 0.20574162679425836 }, "terminations": { "purchase": 209 } }, "failure": { "n": 47, "tasks": 12, "metrics": { "model_steps": { "n": 47, "mean": 11.25531914893617, "median": 8.0, "p90": 22.799999999999997 }, "actions": { "n": 47, "mean": 10.808510638297872, "median": 8.0, "p90": 21.799999999999997 }, "valid_actions": { "n": 47, "mean": 9.446808510638299, "median": 8.0, "p90": 16.0 }, "tokens_per_turn": { "n": 47, "mean": 219.60499527529444, "median": 222.125, "p90": 262.5805555555556 }, "generated_tokens": { "n": 47, "mean": 2310.340425531915, "median": 1999.0, "p90": 4411.0 }, "first_turn_tokens": { "n": 47, "mean": 128.10638297872342, "median": 117.0, "p90": 159.19999999999993 }, "searches": { "n": 47, "mean": 2.3617021276595747, "median": 2.0, "p90": 5.0 }, "unique_queries": { "n": 47, "mean": 2.3404255319148937, "median": 2.0, "p90": 5.0 }, "product_opens": { "n": 47, "mean": 2.0, "median": 2.0, "p90": 4.0 }, "unique_products": { "n": 47, "mean": 1.3404255319148937, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 47, "mean": 0.6595744680851063, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 47, "mean": 1.3617021276595744, "median": 1.0, "p90": 4.0 }, "pagination": { "n": 47, "mean": 0.3617021276595745, "median": 0.0, "p90": 2.0 }, "option_selections": { "n": 47, "mean": 2.595744680851064, "median": 2.0, "p90": 4.399999999999999 }, "option_changes": { "n": 47, "mean": 0.3404255319148936, "median": 0.0, "p90": 0.3999999999999986 }, "protocol_errors": { "n": 47, "mean": 0.2127659574468085, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 47, "mean": 1.3617021276595744, "median": 0.0, "p90": 6.0 }, "reasoning_chars_per_turn": { "n": 47, "mean": 282.6860597454529, "median": 279.5, "p90": 345.05555555555554 }, "first_product_rank": { "n": 47, "mean": 1.1702127659574468, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.03723404255319149, "pooled_tokens_per_turn": 205.2665406427221, "token_complete_episodes": 47, "rates": { "protocol_errors": 0.14893617021276595, "product_revisits": 0.44680851063829785, "option_changes": 0.10638297872340426, "backtracks": 0.6170212765957447 }, "terminations": { "purchase": 36, "generation_length": 11 } } }, { "step": 47, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.796875, "median": 4.0, "p90": 5.5 }, "actions": { "n": 256, "mean": 4.78125, "median": 4.0, "p90": 5.5 }, "valid_actions": { "n": 256, "mean": 4.77734375, "median": 4.0, "p90": 5.5 }, "tokens_per_turn": { "n": 256, "mean": 173.48915336731284, "median": 166.125, "p90": 226.375 }, "generated_tokens": { "n": 256, "mean": 839.32421875, "median": 745.0, "p90": 1071.0 }, "first_turn_tokens": { "n": 256, "mean": 125.640625, "median": 116.0, "p90": 173.0 }, "searches": { "n": 256, "mean": 1.08203125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.078125, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.08203125, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0390625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.08203125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.53125, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 211.45464572192515, "median": 205.875, "p90": 287.55 }, "first_product_rank": { "n": 256, "mean": 1.46875, "median": 1.0, "p90": 1.5 } }, "success_rate": 0.9296875, "reward_mean": 0.948828125, "pooled_tokens_per_turn": 174.97312703583063, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.01171875, "product_revisits": 0.03515625, "option_changes": 0.04296875, "backtracks": 0.05859375 }, "terminations": { "purchase": 256 } }, "success": { "n": 238, "tasks": 31, "metrics": { "model_steps": { "n": 238, "mean": 4.815126050420168, "median": 4.0, "p90": 6.0 }, "actions": { "n": 238, "mean": 4.798319327731092, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 238, "mean": 4.794117647058823, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 238, "mean": 172.45555992450454, "median": 165.0, "p90": 226.22500000000002 }, "generated_tokens": { "n": 238, "mean": 838.7689075630252, "median": 735.5, "p90": 1076.7 }, "first_turn_tokens": { "n": 238, "mean": 124.50840336134453, "median": 115.0, "p90": 171.60000000000002 }, "searches": { "n": 238, "mean": 1.088235294117647, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 238, "mean": 1.084033613445378, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 238, "mean": 1.088235294117647, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 238, "mean": 1.0420168067226891, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 238, "mean": 0.046218487394957986, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 238, "mean": 0.08823529411764706, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 238, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 238, "mean": 1.5294117647058822, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 238, "mean": 0.05042016806722689, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 238, "mean": 0.01680672268907563, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 238, "mean": 0.004201680672268907, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 238, "mean": 209.5379802723228, "median": 200.875, "p90": 286.2675 }, "first_product_rank": { "n": 238, "mean": 1.495798319327731, "median": 1.0, "p90": 1.3000000000000114 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 174.19458987783594, "token_complete_episodes": 238, "rates": { "protocol_errors": 0.012605042016806723, "product_revisits": 0.037815126050420166, "option_changes": 0.046218487394957986, "backtracks": 0.06302521008403361 }, "terminations": { "purchase": 238 } }, "failure": { "n": 18, "tasks": 5, "metrics": { "model_steps": { "n": 18, "mean": 4.555555555555555, "median": 5.0, "p90": 5.0 }, "actions": { "n": 18, "mean": 4.555555555555555, "median": 5.0, "p90": 5.0 }, "valid_actions": { "n": 18, "mean": 4.555555555555555, "median": 5.0, "p90": 5.0 }, "tokens_per_turn": { "n": 18, "mean": 187.15555555555557, "median": 180.7, "p90": 220.175 }, "generated_tokens": { "n": 18, "mean": 846.6666666666666, "median": 854.0, "p90": 986.6 }, "first_turn_tokens": { "n": 18, "mean": 140.61111111111111, "median": 138.0, "p90": 199.4 }, "searches": { "n": 18, "mean": 1.0, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 18, "mean": 1.0, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 18, "mean": 1.0, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 18, "mean": 1.0, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 18, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 18, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 18, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 18, "mean": 1.5555555555555556, "median": 2.0, "p90": 2.0 }, "option_changes": { "n": 18, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 18, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 18, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 18, "mean": 236.79722222222225, "median": 231.8, "p90": 291.925 }, "first_product_rank": { "n": 18, "mean": 1.1111111111111112, "median": 1.0, "p90": 1.3000000000000007 } }, "success_rate": 0.0, "reward_mean": 0.27222222222222225, "pooled_tokens_per_turn": 185.85365853658536, "token_complete_episodes": 18, "rates": { "protocol_errors": 0.0, "product_revisits": 0.0, "option_changes": 0.0, "backtracks": 0.0 }, "terminations": { "purchase": 18 } } }, { "step": 48, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.78515625, "median": 4.0, "p90": 9.0 }, "actions": { "n": 256, "mean": 5.67578125, "median": 4.0, "p90": 9.0 }, "valid_actions": { "n": 256, "mean": 5.30859375, "median": 4.0, "p90": 8.5 }, "tokens_per_turn": { "n": 256, "mean": 202.03400585643158, "median": 181.125, "p90": 272.65 }, "generated_tokens": { "n": 256, "mean": 1222.6875, "median": 771.0, "p90": 3014.5 }, "first_turn_tokens": { "n": 256, "mean": 125.54296875, "median": 118.0, "p90": 166.5 }, "searches": { "n": 256, "mean": 1.30859375, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.296875, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.26171875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.04296875, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.21875, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.30859375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.50390625, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.08203125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.3671875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 240.8960873575174, "median": 225.0, "p90": 348.875 }, "first_product_rank": { "n": 256, "mean": 1.09375, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.796875, "reward_mean": 0.8125, "pooled_tokens_per_turn": 211.34908845374747, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0234375, "product_revisits": 0.11328125, "option_changes": 0.03125, "backtracks": 0.12890625 }, "terminations": { "purchase": 235, "generation_length": 21 } }, "success": { "n": 204, "tasks": 27, "metrics": { "model_steps": { "n": 204, "mean": 4.436274509803922, "median": 4.0, "p90": 5.0 }, "actions": { "n": 204, "mean": 4.421568627450981, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 204, "mean": 4.421568627450981, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 204, "mean": 184.52526065981948, "median": 175.375, "p90": 263.41 }, "generated_tokens": { "n": 204, "mean": 832.5196078431372, "median": 734.0, "p90": 1111.7 }, "first_turn_tokens": { "n": 204, "mean": 125.12745098039215, "median": 117.0, "p90": 165.10000000000005 }, "searches": { "n": 204, "mean": 1.0392156862745099, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 204, "mean": 1.0392156862745099, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 204, "mean": 1.0343137254901962, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 204, "mean": 1.0, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 204, "mean": 0.03431372549019608, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 204, "mean": 0.0392156862745098, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 204, "mean": 0.004901960784313725, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 204, "mean": 1.303921568627451, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 204, "mean": 0.029411764705882353, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 204, "mean": 0.014705882352941176, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 204, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 204, "mean": 228.51214207905383, "median": 211.675, "p90": 330.37500000000006 }, "first_product_rank": { "n": 204, "mean": 1.107843137254902, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 187.66187845303867, "token_complete_episodes": 204, "rates": { "protocol_errors": 0.014705882352941176, "product_revisits": 0.029411764705882353, "option_changes": 0.00980392156862745, "backtracks": 0.029411764705882353 }, "terminations": { "purchase": 204 } }, "failure": { "n": 52, "tasks": 11, "metrics": { "model_steps": { "n": 52, "mean": 11.076923076923077, "median": 8.5, "p90": 25.39999999999999 }, "actions": { "n": 52, "mean": 10.596153846153847, "median": 8.5, "p90": 24.39999999999999 }, "valid_actions": { "n": 52, "mean": 8.788461538461538, "median": 7.5, "p90": 17.0 }, "tokens_per_turn": { "n": 52, "mean": 270.7221600892944, "median": 216.15, "p90": 365.67619047619047 }, "generated_tokens": { "n": 52, "mean": 2753.346153846154, "median": 2558.0, "p90": 4968.3 }, "first_turn_tokens": { "n": 52, "mean": 127.17307692307692, "median": 124.0, "p90": 166.7 }, "searches": { "n": 52, "mean": 2.3653846153846154, "median": 2.0, "p90": 5.0 }, "unique_queries": { "n": 52, "mean": 2.3076923076923075, "median": 2.0, "p90": 5.0 }, "product_opens": { "n": 52, "mean": 2.1538461538461537, "median": 2.0, "p90": 4.0 }, "unique_products": { "n": 52, "mean": 1.2115384615384615, "median": 1.0, "p90": 1.8999999999999986 }, "product_revisits": { "n": 52, "mean": 0.9423076923076923, "median": 0.0, "p90": 3.0 }, "backtracks": { "n": 52, "mean": 1.3653846153846154, "median": 1.0, "p90": 4.0 }, "pagination": { "n": 52, "mean": 0.019230769230769232, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 52, "mean": 2.2884615384615383, "median": 2.0, "p90": 4.0 }, "option_changes": { "n": 52, "mean": 0.28846153846153844, "median": 0.0, "p90": 0.8999999999999986 }, "protocol_errors": { "n": 52, "mean": 0.07692307692307693, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 52, "mean": 1.8076923076923077, "median": 0.0, "p90": 8.0 }, "reasoning_chars_per_turn": { "n": 52, "mean": 289.47925729610506, "median": 258.525, "p90": 444.83473684210526 }, "first_product_rank": { "n": 52, "mean": 1.0384615384615385, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.07692307692307693, "pooled_tokens_per_turn": 248.56597222222223, "token_complete_episodes": 52, "rates": { "protocol_errors": 0.057692307692307696, "product_revisits": 0.4423076923076923, "option_changes": 0.11538461538461539, "backtracks": 0.5192307692307693 }, "terminations": { "generation_length": 21, "purchase": 31 } } }, { "step": 49, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.37109375, "median": 4.0, "p90": 13.0 }, "actions": { "n": 256, "mean": 6.25, "median": 4.0, "p90": 12.5 }, "valid_actions": { "n": 256, "mean": 6.2265625, "median": 4.0, "p90": 12.5 }, "tokens_per_turn": { "n": 256, "mean": 195.8546488477788, "median": 185.46875, "p90": 268.5 }, "generated_tokens": { "n": 256, "mean": 1315.99609375, "median": 838.0, "p90": 2843.0 }, "first_turn_tokens": { "n": 256, "mean": 119.109375, "median": 113.0, "p90": 166.0 }, "searches": { "n": 256, "mean": 1.49609375, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 256, "mean": 1.48046875, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 256, "mean": 1.453125, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 256, "mean": 1.2265625, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.2265625, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.50390625, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.7734375, "median": 1.0, "p90": 4.0 }, "option_changes": { "n": 256, "mean": 0.15625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 245.19722134952917, "median": 229.75, "p90": 363.125 }, "first_product_rank": { "n": 256, "mean": 1.34375, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.78125, "reward_mean": 0.7972005208333333, "pooled_tokens_per_turn": 206.5573267933783, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.05859375, "product_revisits": 0.15234375, "option_changes": 0.05859375, "backtracks": 0.2265625 }, "terminations": { "purchase": 245, "generation_length": 11 } }, "success": { "n": 200, "tasks": 29, "metrics": { "model_steps": { "n": 200, "mean": 5.07, "median": 4.0, "p90": 8.0 }, "actions": { "n": 200, "mean": 5.035, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 200, "mean": 5.025, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 200, "mean": 187.03881220576807, "median": 176.05, "p90": 249.29999999999998 }, "generated_tokens": { "n": 200, "mean": 967.95, "median": 776.0, "p90": 1885.2999999999997 }, "first_turn_tokens": { "n": 200, "mean": 120.78, "median": 111.0, "p90": 173.2 }, "searches": { "n": 200, "mean": 1.19, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 200, "mean": 1.19, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 200, "mean": 1.18, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 200, "mean": 1.07, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 200, "mean": 0.11, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 200, "mean": 0.19, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 200, "mean": 0.02, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 200, "mean": 1.445, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 200, "mean": 0.045, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 200, "mean": 0.035, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 200, "mean": 0.01, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 200, "mean": 233.19648990470313, "median": 220.375, "p90": 322.2075 }, "first_product_rank": { "n": 200, "mean": 1.25, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 190.9171597633136, "token_complete_episodes": 200, "rates": { "protocol_errors": 0.03, "product_revisits": 0.075, "option_changes": 0.03, "backtracks": 0.125 }, "terminations": { "purchase": 200 } }, "failure": { "n": 56, "tasks": 11, "metrics": { "model_steps": { "n": 56, "mean": 11.017857142857142, "median": 10.0, "p90": 20.0 }, "actions": { "n": 56, "mean": 10.589285714285714, "median": 10.0, "p90": 19.0 }, "valid_actions": { "n": 56, "mean": 10.517857142857142, "median": 10.0, "p90": 18.5 }, "tokens_per_turn": { "n": 56, "mean": 227.33977971210274, "median": 211.25, "p90": 333.23636363636365 }, "generated_tokens": { "n": 56, "mean": 2559.0178571428573, "median": 2060.0, "p90": 5059.5 }, "first_turn_tokens": { "n": 56, "mean": 113.14285714285714, "median": 113.0, "p90": 145.0 }, "searches": { "n": 56, "mean": 2.5892857142857144, "median": 2.0, "p90": 5.0 }, "unique_queries": { "n": 56, "mean": 2.517857142857143, "median": 2.0, "p90": 5.0 }, "product_opens": { "n": 56, "mean": 2.4285714285714284, "median": 2.0, "p90": 5.0 }, "unique_products": { "n": 56, "mean": 1.7857142857142858, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 56, "mean": 0.6428571428571429, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 56, "mean": 1.625, "median": 1.0, "p90": 4.0 }, "pagination": { "n": 56, "mean": 0.125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 56, "mean": 2.9464285714285716, "median": 2.0, "p90": 5.0 }, "option_changes": { "n": 56, "mean": 0.5535714285714286, "median": 0.0, "p90": 2.0 }, "protocol_errors": { "n": 56, "mean": 0.23214285714285715, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 56, "mean": 0.07142857142857142, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 56, "mean": 288.0569765096221, "median": 266.0, "p90": 464.03939393939396 }, "first_product_rank": { "n": 56, "mean": 1.6785714285714286, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.0, "reward_mean": 0.07291666666666666, "pooled_tokens_per_turn": 232.2609400324149, "token_complete_episodes": 56, "rates": { "protocol_errors": 0.16071428571428573, "product_revisits": 0.42857142857142855, "option_changes": 0.16071428571428573, "backtracks": 0.5892857142857143 }, "terminations": { "purchase": 45, "generation_length": 11 } } }, { "step": 50, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.6953125, "median": 4.0, "p90": 10.0 }, "actions": { "n": 256, "mean": 5.5703125, "median": 4.0, "p90": 10.0 }, "valid_actions": { "n": 256, "mean": 5.55859375, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 256, "mean": 174.55007871867167, "median": 160.325, "p90": 236.60000000000002 }, "generated_tokens": { "n": 256, "mean": 991.359375, "median": 726.5, "p90": 1951.5 }, "first_turn_tokens": { "n": 256, "mean": 118.421875, "median": 114.0, "p90": 153.0 }, "searches": { "n": 256, "mean": 1.2890625, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.2734375, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.21875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.1328125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.0859375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.2890625, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.09765625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.73046875, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.2421875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.05859375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 207.69375040196394, "median": 191.125, "p90": 299.625 }, "first_product_rank": { "n": 255, "mean": 1.592156862745098, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.83203125, "reward_mean": 0.83203125, "pooled_tokens_per_turn": 174.06584362139918, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.05078125, "product_revisits": 0.06640625, "option_changes": 0.078125, "backtracks": 0.1328125 }, "terminations": { "purchase": 239, "generation_length": 17 } }, "success": { "n": 213, "tasks": 30, "metrics": { "model_steps": { "n": 213, "mean": 4.746478873239437, "median": 4.0, "p90": 5.0 }, "actions": { "n": 213, "mean": 4.732394366197183, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 213, "mean": 4.732394366197183, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 213, "mean": 166.8486771191618, "median": 157.0, "p90": 227.76000000000005 }, "generated_tokens": { "n": 213, "mean": 795.4553990610328, "median": 693.0, "p90": 1114.2000000000003 }, "first_turn_tokens": { "n": 213, "mean": 120.77934272300469, "median": 116.0, "p90": 155.0 }, "searches": { "n": 213, "mean": 1.0657276995305165, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 213, "mean": 1.0469483568075117, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 213, "mean": 1.0610328638497653, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 213, "mean": 1.0187793427230047, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 213, "mean": 0.04225352112676056, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 213, "mean": 0.06572769953051644, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 213, "mean": 0.004694835680751174, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 213, "mean": 1.5352112676056338, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 213, "mean": 0.06572769953051644, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 213, "mean": 0.014084507042253521, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 213, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 213, "mean": 200.8774143673688, "median": 185.6, "p90": 292.1 }, "first_product_rank": { "n": 213, "mean": 1.1549295774647887, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 167.58852621167162, "token_complete_episodes": 213, "rates": { "protocol_errors": 0.014084507042253521, "product_revisits": 0.04225352112676056, "option_changes": 0.028169014084507043, "backtracks": 0.06103286384976526 }, "terminations": { "purchase": 213 } }, "failure": { "n": 43, "tasks": 8, "metrics": { "model_steps": { "n": 43, "mean": 10.395348837209303, "median": 7.0, "p90": 18.0 }, "actions": { "n": 43, "mean": 9.720930232558139, "median": 7.0, "p90": 17.0 }, "valid_actions": { "n": 43, "mean": 9.651162790697674, "median": 7.0, "p90": 17.0 }, "tokens_per_turn": { "n": 43, "mean": 212.69888199066236, "median": 182.75, "p90": 260.85 }, "generated_tokens": { "n": 43, "mean": 1961.7674418604652, "median": 1930.0, "p90": 3186.0000000000005 }, "first_turn_tokens": { "n": 43, "mean": 106.74418604651163, "median": 102.0, "p90": 126.00000000000003 }, "searches": { "n": 43, "mean": 2.395348837209302, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 43, "mean": 2.395348837209302, "median": 1.0, "p90": 5.0 }, "product_opens": { "n": 43, "mean": 2.0, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 43, "mean": 1.697674418604651, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 43, "mean": 0.3023255813953488, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 43, "mean": 1.3953488372093024, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 43, "mean": 0.5581395348837209, "median": 0.0, "p90": 2.8000000000000043 }, "option_selections": { "n": 43, "mean": 2.697674418604651, "median": 2.0, "p90": 6.0 }, "option_changes": { "n": 43, "mean": 1.1162790697674418, "median": 0.0, "p90": 3.8000000000000043 }, "protocol_errors": { "n": 43, "mean": 0.27906976744186046, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 43, "mean": 0.06976744186046512, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 43, "mean": 241.45839168960973, "median": 235.6, "p90": 334.68846153846164 }, "first_product_rank": { "n": 42, "mean": 3.8095238095238093, "median": 1.0, "p90": 7.0 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 188.7158836689038, "token_complete_episodes": 43, "rates": { "protocol_errors": 0.23255813953488372, "product_revisits": 0.18604651162790697, "option_changes": 0.32558139534883723, "backtracks": 0.4883720930232558 }, "terminations": { "purchase": 26, "generation_length": 17 } } }, { "step": 51, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.74609375, "median": 4.0, "p90": 9.0 }, "actions": { "n": 256, "mean": 5.63671875, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.49609375, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 181.25643639280784, "median": 175.1875, "p90": 248.0 }, "generated_tokens": { "n": 256, "mean": 1095.6171875, "median": 781.0, "p90": 1821.5 }, "first_turn_tokens": { "n": 256, "mean": 121.0546875, "median": 113.0, "p90": 163.0 }, "searches": { "n": 256, "mean": 1.32421875, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.3203125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.28515625, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.10546875, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.1796875, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.32421875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.59765625, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.1640625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.07421875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.140625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 225.3272696795298, "median": 216.0340909090909, "p90": 331.0272727272727 }, "first_product_rank": { "n": 256, "mean": 1.3984375, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.8203125, "reward_mean": 0.8260091145833334, "pooled_tokens_per_turn": 190.6716519374575, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0546875, "product_revisits": 0.12890625, "option_changes": 0.05078125, "backtracks": 0.18359375 }, "terminations": { "purchase": 246, "generation_length": 9, "action_limit": 1 } }, "success": { "n": 210, "tasks": 30, "metrics": { "model_steps": { "n": 210, "mean": 4.995238095238095, "median": 4.0, "p90": 8.0 }, "actions": { "n": 210, "mean": 4.961904761904762, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 210, "mean": 4.947619047619048, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 210, "mean": 176.8608298447584, "median": 168.5, "p90": 243.79999999999998 }, "generated_tokens": { "n": 210, "mean": 896.2333333333333, "median": 750.5, "p90": 1498.4999999999995 }, "first_turn_tokens": { "n": 210, "mean": 122.35714285714286, "median": 115.0, "p90": 165.39999999999998 }, "searches": { "n": 210, "mean": 1.1714285714285715, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 210, "mean": 1.1666666666666667, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 210, "mean": 1.1714285714285715, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 210, "mean": 1.061904761904762, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 210, "mean": 0.10952380952380952, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 210, "mean": 0.17142857142857143, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 210, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 210, "mean": 1.4333333333333333, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 210, "mean": 0.04285714285714286, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 210, "mean": 0.03333333333333333, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 210, "mean": 0.014285714285714285, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 210, "mean": 218.2940372788587, "median": 211.25, "p90": 306.12499999999994 }, "first_product_rank": { "n": 210, "mean": 1.2095238095238094, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 179.41754051477596, "token_complete_episodes": 210, "rates": { "protocol_errors": 0.03333333333333333, "product_revisits": 0.10476190476190476, "option_changes": 0.0380952380952381, "backtracks": 0.1523809523809524 }, "terminations": { "purchase": 210 } }, "failure": { "n": 46, "tasks": 10, "metrics": { "model_steps": { "n": 46, "mean": 9.173913043478262, "median": 5.0, "p90": 23.0 }, "actions": { "n": 46, "mean": 8.717391304347826, "median": 5.0, "p90": 21.5 }, "valid_actions": { "n": 46, "mean": 8.0, "median": 5.0, "p90": 17.0 }, "tokens_per_turn": { "n": 46, "mean": 201.32333585129425, "median": 199.47, "p90": 264.75 }, "generated_tokens": { "n": 46, "mean": 2005.8478260869565, "median": 965.5, "p90": 4914.5 }, "first_turn_tokens": { "n": 46, "mean": 115.1086956521739, "median": 109.0, "p90": 149.0 }, "searches": { "n": 46, "mean": 2.0217391304347827, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 46, "mean": 2.0217391304347827, "median": 1.0, "p90": 5.0 }, "product_opens": { "n": 46, "mean": 1.8043478260869565, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 46, "mean": 1.3043478260869565, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 46, "mean": 0.5, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 46, "mean": 1.0217391304347827, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 46, "mean": 0.021739130434782608, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 46, "mean": 2.347826086956522, "median": 1.0, "p90": 4.0 }, "option_changes": { "n": 46, "mean": 0.717391304347826, "median": 0.0, "p90": 0.5 }, "protocol_errors": { "n": 46, "mean": 0.2608695652173913, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 46, "mean": 0.717391304347826, "median": 0.0, "p90": 4.0 }, "reasoning_chars_per_turn": { "n": 46, "mean": 257.43550455215865, "median": 252.41477272727275, "p90": 365.0673076923077 }, "first_product_rank": { "n": 46, "mean": 2.260869565217391, "median": 1.0, "p90": 5.5 } }, "success_rate": 0.0, "reward_mean": 0.03170289855072463, "pooled_tokens_per_turn": 218.64691943127963, "token_complete_episodes": 46, "rates": { "protocol_errors": 0.15217391304347827, "product_revisits": 0.2391304347826087, "option_changes": 0.10869565217391304, "backtracks": 0.32608695652173914 }, "terminations": { "purchase": 36, "generation_length": 9, "action_limit": 1 } } }, { "step": 52, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.29296875, "median": 4.0, "p90": 12.0 }, "actions": { "n": 256, "mean": 6.171875, "median": 4.0, "p90": 12.0 }, "valid_actions": { "n": 256, "mean": 6.16015625, "median": 4.0, "p90": 12.0 }, "tokens_per_turn": { "n": 256, "mean": 183.7131166219285, "median": 179.94642857142856, "p90": 243.01111111111112 }, "generated_tokens": { "n": 256, "mean": 1219.953125, "median": 800.5, "p90": 2352.5 }, "first_turn_tokens": { "n": 256, "mean": 127.0859375, "median": 124.0, "p90": 169.5 }, "searches": { "n": 256, "mean": 1.4453125, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 256, "mean": 1.421875, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 256, "mean": 1.3828125, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 256, "mean": 1.12109375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.26171875, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.4453125, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.8515625, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.16796875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 230.05934416285857, "median": 222.02777777777777, "p90": 326.65277777777777 }, "first_product_rank": { "n": 256, "mean": 2.015625, "median": 1.0, "p90": 4.0 } }, "success_rate": 0.8046875, "reward_mean": 0.8046875, "pooled_tokens_per_turn": 193.8597144630664, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.06640625, "product_revisits": 0.171875, "option_changes": 0.0546875, "backtracks": 0.21875 }, "terminations": { "purchase": 245, "generation_length": 11 } }, "success": { "n": 206, "tasks": 29, "metrics": { "model_steps": { "n": 206, "mean": 5.349514563106796, "median": 4.0, "p90": 8.5 }, "actions": { "n": 206, "mean": 5.300970873786408, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 206, "mean": 5.29126213592233, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 206, "mean": 178.18027614914752, "median": 173.875, "p90": 240.25 }, "generated_tokens": { "n": 206, "mean": 991.6844660194175, "median": 768.0, "p90": 1728.0 }, "first_turn_tokens": { "n": 206, "mean": 128.70388349514562, "median": 124.5, "p90": 172.5 }, "searches": { "n": 206, "mean": 1.2184466019417475, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 206, "mean": 1.1941747572815533, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 206, "mean": 1.203883495145631, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 206, "mean": 1.0388349514563107, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 206, "mean": 0.1650485436893204, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 206, "mean": 0.21844660194174756, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 206, "mean": 0.019417475728155338, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 206, "mean": 1.6310679611650485, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 206, "mean": 0.11165048543689321, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 206, "mean": 0.04854368932038835, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 206, "mean": 0.009708737864077669, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 206, "mean": 219.80023448773446, "median": 211.0, "p90": 306.3571428571429 }, "first_product_rank": { "n": 206, "mean": 1.4368932038834952, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 185.37840290381126, "token_complete_episodes": 206, "rates": { "protocol_errors": 0.043689320388349516, "product_revisits": 0.1262135922330097, "option_changes": 0.04854368932038835, "backtracks": 0.14563106796116504 }, "terminations": { "purchase": 206 } }, "failure": { "n": 50, "tasks": 11, "metrics": { "model_steps": { "n": 50, "mean": 10.18, "median": 8.0, "p90": 21.1 }, "actions": { "n": 50, "mean": 9.76, "median": 8.0, "p90": 20.1 }, "valid_actions": { "n": 50, "mean": 9.74, "median": 8.0, "p90": 19.200000000000003 }, "tokens_per_turn": { "n": 50, "mean": 206.50841936978605, "median": 204.93269230769232, "p90": 254.61447368421054 }, "generated_tokens": { "n": 50, "mean": 2160.42, "median": 1575.0, "p90": 5016.3 }, "first_turn_tokens": { "n": 50, "mean": 120.42, "median": 120.5, "p90": 147.2 }, "searches": { "n": 50, "mean": 2.38, "median": 2.0, "p90": 5.0 }, "unique_queries": { "n": 50, "mean": 2.36, "median": 2.0, "p90": 5.0 }, "product_opens": { "n": 50, "mean": 2.12, "median": 1.5, "p90": 4.100000000000001 }, "unique_products": { "n": 50, "mean": 1.46, "median": 1.0, "p90": 2.1000000000000014 }, "product_revisits": { "n": 50, "mean": 0.66, "median": 0.0, "p90": 2.1000000000000014 }, "backtracks": { "n": 50, "mean": 1.38, "median": 1.0, "p90": 4.0 }, "pagination": { "n": 50, "mean": 0.32, "median": 0.0, "p90": 1.1000000000000014 }, "option_selections": { "n": 50, "mean": 2.76, "median": 2.0, "p90": 6.100000000000001 }, "option_changes": { "n": 50, "mean": 0.4, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 50, "mean": 0.2, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 50, "mean": 0.02, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 50, "mean": 272.32687602436977, "median": 274.25, "p90": 341.30386904761906 }, "first_product_rank": { "n": 50, "mean": 4.4, "median": 2.0, "p90": 13.200000000000003 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 212.2220039292731, "token_complete_episodes": 50, "rates": { "protocol_errors": 0.16, "product_revisits": 0.36, "option_changes": 0.08, "backtracks": 0.52 }, "terminations": { "purchase": 39, "generation_length": 11 } } }, { "step": 53, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.23046875, "median": 4.0, "p90": 12.0 }, "actions": { "n": 256, "mean": 6.1328125, "median": 4.0, "p90": 12.0 }, "valid_actions": { "n": 256, "mean": 6.109375, "median": 4.0, "p90": 12.0 }, "tokens_per_turn": { "n": 256, "mean": 171.30559430772712, "median": 165.0, "p90": 223.89772727272725 }, "generated_tokens": { "n": 256, "mean": 1127.015625, "median": 751.5, "p90": 2204.0 }, "first_turn_tokens": { "n": 256, "mean": 124.34375, "median": 119.0, "p90": 165.5 }, "searches": { "n": 256, "mean": 1.44140625, "median": 1.0, "p90": 2.5 }, "unique_queries": { "n": 256, "mean": 1.42578125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.390625, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.1328125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.2578125, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.4453125, "median": 0.0, "p90": 1.5 }, "pagination": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.83984375, "median": 1.0, "p90": 4.0 }, "option_changes": { "n": 256, "mean": 0.17578125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.05859375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 211.00068078321968, "median": 199.15, "p90": 294.89473684210526 }, "first_product_rank": { "n": 256, "mean": 1.64453125, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.81640625, "reward_mean": 0.8546781994047619, "pooled_tokens_per_turn": 180.88777429467083, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0546875, "product_revisits": 0.16796875, "option_changes": 0.09375, "backtracks": 0.21484375 }, "terminations": { "purchase": 246, "generation_length": 10 } }, "success": { "n": 209, "tasks": 29, "metrics": { "model_steps": { "n": 209, "mean": 5.2105263157894735, "median": 4.0, "p90": 8.0 }, "actions": { "n": 209, "mean": 5.177033492822966, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 209, "mean": 5.172248803827751, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 209, "mean": 166.83017408666487, "median": 159.8, "p90": 221.8 }, "generated_tokens": { "n": 209, "mean": 896.8755980861245, "median": 716.0, "p90": 1536.4000000000003 }, "first_turn_tokens": { "n": 209, "mean": 125.41626794258373, "median": 120.0, "p90": 171.0 }, "searches": { "n": 209, "mean": 1.2200956937799043, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 209, "mean": 1.2057416267942584, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 209, "mean": 1.2105263157894737, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 209, "mean": 1.062200956937799, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 209, "mean": 0.14832535885167464, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 209, "mean": 0.22488038277511962, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 209, "mean": 0.014354066985645933, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 209, "mean": 1.5023923444976077, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 209, "mean": 0.06220095693779904, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 209, "mean": 0.03349282296650718, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 209, "mean": 0.004784688995215311, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 209, "mean": 205.35563997406106, "median": 193.5, "p90": 288.50000000000006 }, "first_product_rank": { "n": 209, "mean": 1.5023923444976077, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 172.12764003673095, "token_complete_episodes": 209, "rates": { "protocol_errors": 0.028708133971291867, "product_revisits": 0.12440191387559808, "option_changes": 0.04784688995215311, "backtracks": 0.13875598086124402 }, "terminations": { "purchase": 209 } }, "failure": { "n": 47, "tasks": 11, "metrics": { "model_steps": { "n": 47, "mean": 10.76595744680851, "median": 8.0, "p90": 22.0 }, "actions": { "n": 47, "mean": 10.382978723404255, "median": 8.0, "p90": 21.0 }, "valid_actions": { "n": 47, "mean": 10.27659574468085, "median": 8.0, "p90": 20.4 }, "tokens_per_turn": { "n": 47, "mean": 191.20693103542928, "median": 179.17391304347825, "p90": 235.63333333333333 }, "generated_tokens": { "n": 47, "mean": 2150.404255319149, "median": 1442.0, "p90": 4218.599999999999 }, "first_turn_tokens": { "n": 47, "mean": 119.57446808510639, "median": 114.0, "p90": 145.99999999999997 }, "searches": { "n": 47, "mean": 2.425531914893617, "median": 2.0, "p90": 5.0 }, "unique_queries": { "n": 47, "mean": 2.404255319148936, "median": 2.0, "p90": 5.0 }, "product_opens": { "n": 47, "mean": 2.1914893617021276, "median": 2.0, "p90": 4.0 }, "unique_products": { "n": 47, "mean": 1.446808510638298, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 47, "mean": 0.7446808510638298, "median": 0.0, "p90": 2.3999999999999986 }, "backtracks": { "n": 47, "mean": 1.425531914893617, "median": 1.0, "p90": 4.0 }, "pagination": { "n": 47, "mean": 0.10638297872340426, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 47, "mean": 3.3404255319148937, "median": 3.0, "p90": 6.399999999999999 }, "option_changes": { "n": 47, "mean": 0.6808510638297872, "median": 0.0, "p90": 2.3999999999999986 }, "protocol_errors": { "n": 47, "mean": 0.1702127659574468, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 47, "mean": 0.10638297872340426, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 47, "mean": 236.10309629628685, "median": 222.0, "p90": 317.5844155844156 }, "first_product_rank": { "n": 47, "mean": 2.276595744680851, "median": 1.0, "p90": 6.399999999999999 } }, "success_rate": 0.0, "reward_mean": 0.20845997973657548, "pooled_tokens_per_turn": 199.7411067193676, "token_complete_episodes": 47, "rates": { "protocol_errors": 0.1702127659574468, "product_revisits": 0.3617021276595745, "option_changes": 0.2978723404255319, "backtracks": 0.5531914893617021 }, "terminations": { "purchase": 37, "generation_length": 10 } } }, { "step": 54, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.97265625, "median": 4.0, "p90": 10.0 }, "actions": { "n": 256, "mean": 5.84765625, "median": 4.0, "p90": 9.5 }, "valid_actions": { "n": 256, "mean": 5.8359375, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 256, "mean": 187.78517889878546, "median": 178.9, "p90": 261.1428571428571 }, "generated_tokens": { "n": 256, "mean": 1158.06640625, "median": 807.0, "p90": 2204.0 }, "first_turn_tokens": { "n": 256, "mean": 129.48828125, "median": 121.0, "p90": 177.5 }, "searches": { "n": 256, "mean": 1.38671875, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.37890625, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.35546875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.109375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.24609375, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.38671875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.703125, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.19140625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.09375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 234.24298326199238, "median": 221.85, "p90": 341.375 }, "first_product_rank": { "n": 255, "mean": 1.2431372549019608, "median": 1.0, "p90": 1.5999999999999943 } }, "success_rate": 0.89453125, "reward_mean": 0.9045758928571428, "pooled_tokens_per_turn": 193.89470241988226, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0703125, "product_revisits": 0.1875, "option_changes": 0.08203125, "backtracks": 0.22265625 }, "terminations": { "purchase": 248, "generation_length": 8 } }, "success": { "n": 229, "tasks": 31, "metrics": { "model_steps": { "n": 229, "mean": 5.292576419213974, "median": 4.0, "p90": 8.0 }, "actions": { "n": 229, "mean": 5.2314410480349345, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 229, "mean": 5.22707423580786, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 229, "mean": 184.09014744709583, "median": 177.25, "p90": 249.62000000000003 }, "generated_tokens": { "n": 229, "mean": 990.240174672489, "median": 771.0, "p90": 1663.200000000001 }, "first_turn_tokens": { "n": 229, "mean": 130.15720524017468, "median": 123.0, "p90": 179.40000000000003 }, "searches": { "n": 229, "mean": 1.2532751091703056, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 229, "mean": 1.2489082969432315, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 229, "mean": 1.240174672489083, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 229, "mean": 1.0480349344978166, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 229, "mean": 0.19213973799126638, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 229, "mean": 0.25327510917030566, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 229, "mean": 0.008733624454148471, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 229, "mean": 1.4716157205240175, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 229, "mean": 0.06550218340611354, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 229, "mean": 0.0611353711790393, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 229, "mean": 0.004366812227074236, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 229, "mean": 229.65545011043983, "median": 219.75, "p90": 328.55 }, "first_product_rank": { "n": 229, "mean": 1.244541484716157, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 187.09983498349834, "token_complete_episodes": 229, "rates": { "protocol_errors": 0.048034934497816595, "product_revisits": 0.1572052401746725, "option_changes": 0.048034934497816595, "backtracks": 0.17467248908296942 }, "terminations": { "purchase": 229 } }, "failure": { "n": 27, "tasks": 7, "metrics": { "model_steps": { "n": 27, "mean": 11.74074074074074, "median": 9.0, "p90": 21.400000000000002 }, "actions": { "n": 27, "mean": 11.074074074074074, "median": 8.0, "p90": 20.400000000000002 }, "valid_actions": { "n": 27, "mean": 11.0, "median": 8.0, "p90": 20.400000000000002 }, "tokens_per_turn": { "n": 27, "mean": 219.1245197297829, "median": 205.95, "p90": 307.90000000000003 }, "generated_tokens": { "n": 27, "mean": 2581.4814814814813, "median": 1904.0, "p90": 4963.800000000002 }, "first_turn_tokens": { "n": 27, "mean": 123.81481481481481, "median": 117.0, "p90": 160.20000000000002 }, "searches": { "n": 27, "mean": 2.5185185185185186, "median": 2.0, "p90": 5.0 }, "unique_queries": { "n": 27, "mean": 2.4814814814814814, "median": 2.0, "p90": 5.0 }, "product_opens": { "n": 27, "mean": 2.3333333333333335, "median": 2.0, "p90": 4.400000000000002 }, "unique_products": { "n": 27, "mean": 1.6296296296296295, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 27, "mean": 0.7037037037037037, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 27, "mean": 1.5185185185185186, "median": 1.0, "p90": 4.0 }, "pagination": { "n": 27, "mean": 0.25925925925925924, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 27, "mean": 3.6666666666666665, "median": 2.0, "p90": 7.400000000000002 }, "option_changes": { "n": 27, "mean": 1.2592592592592593, "median": 0.0, "p90": 4.0 }, "protocol_errors": { "n": 27, "mean": 0.37037037037037035, "median": 0.0, "p90": 1.4000000000000021 }, "invalid_actions": { "n": 27, "mean": 0.07407407407407407, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 27, "mean": 273.1520607325675, "median": 259.8, "p90": 407.909090909091 }, "first_product_rank": { "n": 26, "mean": 1.2307692307692308, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.0, "reward_mean": 0.09523809523809523, "pooled_tokens_per_turn": 219.87381703470032, "token_complete_episodes": 27, "rates": { "protocol_errors": 0.25925925925925924, "product_revisits": 0.4444444444444444, "option_changes": 0.37037037037037035, "backtracks": 0.6296296296296297 }, "terminations": { "purchase": 19, "generation_length": 8 } } }, { "step": 55, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 7.31640625, "median": 4.0, "p90": 17.5 }, "actions": { "n": 256, "mean": 7.015625, "median": 4.0, "p90": 16.0 }, "valid_actions": { "n": 256, "mean": 6.7734375, "median": 4.0, "p90": 15.0 }, "tokens_per_turn": { "n": 256, "mean": 183.08941237954724, "median": 181.1375, "p90": 231.25 }, "generated_tokens": { "n": 256, "mean": 1386.8125, "median": 786.5, "p90": 3406.5 }, "first_turn_tokens": { "n": 256, "mean": 143.25, "median": 134.0, "p90": 205.5 }, "searches": { "n": 256, "mean": 1.6640625, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 256, "mean": 1.62890625, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 256, "mean": 1.50390625, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 256, "mean": 1.1953125, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.30859375, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.66796875, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 256, "mean": 0.20703125, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 256, "mean": 1.84375, "median": 1.0, "p90": 3.5 }, "option_changes": { "n": 256, "mean": 0.2890625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.1875, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.2421875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 232.78041718599906, "median": 227.22222222222223, "p90": 311.875 }, "first_product_rank": { "n": 254, "mean": 1.562992125984252, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.76171875, "reward_mean": 0.7652698863636364, "pooled_tokens_per_turn": 189.5483182060865, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.15234375, "product_revisits": 0.1953125, "option_changes": 0.0546875, "backtracks": 0.28515625 }, "terminations": { "purchase": 227, "generation_length": 29 } }, "success": { "n": 195, "tasks": 27, "metrics": { "model_steps": { "n": 195, "mean": 5.051282051282051, "median": 4.0, "p90": 8.0 }, "actions": { "n": 195, "mean": 4.964102564102564, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 195, "mean": 4.948717948717949, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 195, "mean": 177.4287028356259, "median": 173.0, "p90": 219.92 }, "generated_tokens": { "n": 195, "mean": 918.6974358974359, "median": 749.0, "p90": 1552.3999999999999 }, "first_turn_tokens": { "n": 195, "mean": 137.73333333333332, "median": 128.0, "p90": 196.6 }, "searches": { "n": 195, "mean": 1.1897435897435897, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 195, "mean": 1.1794871794871795, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 195, "mean": 1.1692307692307693, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 195, "mean": 1.0564102564102564, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 195, "mean": 0.11282051282051282, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 195, "mean": 0.18974358974358974, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 195, "mean": 0.06666666666666667, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 195, "mean": 1.3333333333333333, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 195, "mean": 0.13846153846153847, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 195, "mean": 0.08717948717948718, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 195, "mean": 0.015384615384615385, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 195, "mean": 224.67653043537658, "median": 220.5, "p90": 300.0 }, "first_product_rank": { "n": 195, "mean": 1.3384615384615384, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 181.8741116751269, "token_complete_episodes": 195, "rates": { "protocol_errors": 0.08205128205128205, "product_revisits": 0.09230769230769231, "option_changes": 0.020512820512820513, "backtracks": 0.14358974358974358 }, "terminations": { "purchase": 195 } }, "failure": { "n": 61, "tasks": 13, "metrics": { "model_steps": { "n": 61, "mean": 14.557377049180328, "median": 17.0, "p90": 24.0 }, "actions": { "n": 61, "mean": 13.573770491803279, "median": 15.0, "p90": 23.0 }, "valid_actions": { "n": 61, "mean": 12.60655737704918, "median": 14.0, "p90": 20.0 }, "tokens_per_turn": { "n": 61, "mean": 201.18512321667288, "median": 190.8095238095238, "p90": 258.4 }, "generated_tokens": { "n": 61, "mean": 2883.245901639344, "median": 3147.0, "p90": 4658.0 }, "first_turn_tokens": { "n": 61, "mean": 160.88524590163934, "median": 148.0, "p90": 238.0 }, "searches": { "n": 61, "mean": 3.180327868852459, "median": 4.0, "p90": 5.0 }, "unique_queries": { "n": 61, "mean": 3.0655737704918034, "median": 3.0, "p90": 5.0 }, "product_opens": { "n": 61, "mean": 2.5737704918032787, "median": 3.0, "p90": 4.0 }, "unique_products": { "n": 61, "mean": 1.639344262295082, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 61, "mean": 0.9344262295081968, "median": 1.0, "p90": 3.0 }, "backtracks": { "n": 61, "mean": 2.19672131147541, "median": 3.0, "p90": 4.0 }, "pagination": { "n": 61, "mean": 0.6557377049180327, "median": 0.0, "p90": 2.0 }, "option_selections": { "n": 61, "mean": 3.4754098360655736, "median": 3.0, "p90": 7.0 }, "option_changes": { "n": 61, "mean": 0.7704918032786885, "median": 0.0, "p90": 3.0 }, "protocol_errors": { "n": 61, "mean": 0.5081967213114754, "median": 0.0, "p90": 2.0 }, "invalid_actions": { "n": 61, "mean": 0.9672131147540983, "median": 0.0, "p90": 5.0 }, "reasoning_chars_per_turn": { "n": 61, "mean": 258.6862846674971, "median": 240.1764705882353, "p90": 340.0416666666667 }, "first_product_rank": { "n": 59, "mean": 2.305084745762712, "median": 1.0, "p90": 6.0 } }, "success_rate": 0.0, "reward_mean": 0.014903129657228018, "pooled_tokens_per_turn": 198.0608108108108, "token_complete_episodes": 61, "rates": { "protocol_errors": 0.3770491803278688, "product_revisits": 0.5245901639344263, "option_changes": 0.16393442622950818, "backtracks": 0.7377049180327869 }, "terminations": { "generation_length": 29, "purchase": 32 } } }, { "step": 56, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.85546875, "median": 4.0, "p90": 10.5 }, "actions": { "n": 256, "mean": 5.69921875, "median": 4.0, "p90": 10.0 }, "valid_actions": { "n": 256, "mean": 5.65625, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 256, "mean": 175.59517426422323, "median": 171.25, "p90": 223.75 }, "generated_tokens": { "n": 256, "mean": 1039.5, "median": 749.0, "p90": 1980.5 }, "first_turn_tokens": { "n": 256, "mean": 130.1875, "median": 128.0, "p90": 163.0 }, "searches": { "n": 256, "mean": 1.37109375, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.36328125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.265625, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.11328125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.15234375, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.37109375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.15625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.5390625, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.171875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.109375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 218.86582390308254, "median": 209.9671052631579, "p90": 289.0 }, "first_product_rank": { "n": 251, "mean": 1.2191235059760956, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.84765625, "reward_mean": 0.8483072916666666, "pooled_tokens_per_turn": 177.5263509006004, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.09765625, "product_revisits": 0.12890625, "option_changes": 0.0625, "backtracks": 0.18359375 }, "terminations": { "purchase": 244, "generation_length": 12 } }, "success": { "n": 217, "tasks": 30, "metrics": { "model_steps": { "n": 217, "mean": 5.064516129032258, "median": 4.0, "p90": 8.0 }, "actions": { "n": 217, "mean": 5.0046082949308754, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 217, "mean": 4.9631336405529956, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 217, "mean": 174.78267800277493, "median": 169.11111111111111, "p90": 222.375 }, "generated_tokens": { "n": 217, "mean": 896.6912442396314, "median": 726.0, "p90": 1522.0 }, "first_turn_tokens": { "n": 217, "mean": 128.10599078341014, "median": 126.0, "p90": 162.0 }, "searches": { "n": 217, "mean": 1.1612903225806452, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 217, "mean": 1.1612903225806452, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 217, "mean": 1.152073732718894, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 217, "mean": 1.0599078341013826, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 217, "mean": 0.09216589861751152, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 217, "mean": 0.16129032258064516, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 217, "mean": 0.03225806451612903, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 217, "mean": 1.456221198156682, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 217, "mean": 0.14746543778801843, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 217, "mean": 0.059907834101382486, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 217, "mean": 0.041474654377880185, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 217, "mean": 216.98249585043433, "median": 208.0, "p90": 287.8 }, "first_product_rank": { "n": 217, "mean": 1.1336405529953917, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 177.05368516833485, "token_complete_episodes": 217, "rates": { "protocol_errors": 0.059907834101382486, "product_revisits": 0.08294930875576037, "option_changes": 0.055299539170506916, "backtracks": 0.1152073732718894 }, "terminations": { "purchase": 217 } }, "failure": { "n": 39, "tasks": 10, "metrics": { "model_steps": { "n": 39, "mean": 10.256410256410257, "median": 10.0, "p90": 20.0 }, "actions": { "n": 39, "mean": 9.564102564102564, "median": 10.0, "p90": 18.0 }, "valid_actions": { "n": 39, "mean": 9.512820512820513, "median": 10.0, "p90": 18.0 }, "tokens_per_turn": { "n": 39, "mean": 180.1159867958715, "median": 175.54545454545453, "p90": 224.9 }, "generated_tokens": { "n": 39, "mean": 1834.1025641025642, "median": 1449.0, "p90": 3985.4000000000005 }, "first_turn_tokens": { "n": 39, "mean": 141.76923076923077, "median": 137.0, "p90": 167.60000000000002 }, "searches": { "n": 39, "mean": 2.5384615384615383, "median": 2.0, "p90": 5.0 }, "unique_queries": { "n": 39, "mean": 2.4871794871794872, "median": 2.0, "p90": 5.0 }, "product_opens": { "n": 39, "mean": 1.8974358974358974, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 39, "mean": 1.4102564102564104, "median": 1.0, "p90": 2.200000000000003 }, "product_revisits": { "n": 39, "mean": 0.48717948717948717, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 39, "mean": 1.5384615384615385, "median": 1.0, "p90": 4.0 }, "pagination": { "n": 39, "mean": 0.8461538461538461, "median": 0.0, "p90": 4.0 }, "option_selections": { "n": 39, "mean": 2.0, "median": 1.0, "p90": 5.0 }, "option_changes": { "n": 39, "mean": 0.3076923076923077, "median": 0.0, "p90": 0.20000000000000284 }, "protocol_errors": { "n": 39, "mean": 0.38461538461538464, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 39, "mean": 0.05128205128205128, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 39, "mean": 229.34485434986885, "median": 230.63636363636363, "p90": 290.45000000000005 }, "first_product_rank": { "n": 34, "mean": 1.7647058823529411, "median": 1.0, "p90": 3.3999999999999986 } }, "success_rate": 0.0, "reward_mean": 0.004273504273504273, "pooled_tokens_per_turn": 178.825, "token_complete_episodes": 39, "rates": { "protocol_errors": 0.3076923076923077, "product_revisits": 0.38461538461538464, "option_changes": 0.10256410256410256, "backtracks": 0.5641025641025641 }, "terminations": { "purchase": 27, "generation_length": 12 } } }, { "step": 57, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.5703125, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.5234375, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.515625, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 182.39411204030344, "median": 179.125, "p90": 228.5 }, "generated_tokens": { "n": 256, "mean": 832.05078125, "median": 770.0, "p90": 1052.5 }, "first_turn_tokens": { "n": 256, "mean": 144.9375, "median": 140.5, "p90": 198.5 }, "searches": { "n": 256, "mean": 1.078125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.07421875, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.05859375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.03125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.3046875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 226.0900776632222, "median": 224.1153846153846, "p90": 295.625 }, "first_product_rank": { "n": 256, "mean": 1.609375, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.83984375, "reward_mean": 0.83984375, "pooled_tokens_per_turn": 182.05555555555554, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0234375, "product_revisits": 0.02734375, "option_changes": 0.0078125, "backtracks": 0.046875 }, "terminations": { "purchase": 250, "generation_length": 6 } }, "success": { "n": 215, "tasks": 30, "metrics": { "model_steps": { "n": 215, "mean": 4.423255813953489, "median": 4.0, "p90": 5.0 }, "actions": { "n": 215, "mean": 4.404651162790698, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 215, "mean": 4.4, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 215, "mean": 180.32275747508305, "median": 177.0, "p90": 227.26999999999998 }, "generated_tokens": { "n": 215, "mean": 793.7209302325581, "median": 769.0, "p90": 1020.9999999999999 }, "first_turn_tokens": { "n": 215, "mean": 143.7674418604651, "median": 139.0, "p90": 196.6 }, "searches": { "n": 215, "mean": 1.0232558139534884, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 215, "mean": 1.0186046511627906, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 215, "mean": 1.0186046511627906, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 215, "mean": 1.0046511627906978, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 215, "mean": 0.013953488372093023, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 215, "mean": 0.023255813953488372, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 215, "mean": 0.013953488372093023, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 215, "mean": 1.3209302325581396, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 215, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 215, "mean": 0.018604651162790697, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 215, "mean": 0.004651162790697674, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 215, "mean": 222.24844407530455, "median": 215.0, "p90": 294.65 }, "first_product_rank": { "n": 215, "mean": 1.5395348837209302, "median": 1.0, "p90": 1.5999999999999943 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 179.4426919032597, "token_complete_episodes": 215, "rates": { "protocol_errors": 0.018604651162790697, "product_revisits": 0.013953488372093023, "option_changes": 0.0, "backtracks": 0.018604651162790697 }, "terminations": { "purchase": 215 } }, "failure": { "n": 41, "tasks": 8, "metrics": { "model_steps": { "n": 41, "mean": 5.341463414634147, "median": 4.0, "p90": 11.0 }, "actions": { "n": 41, "mean": 5.146341463414634, "median": 4.0, "p90": 10.0 }, "valid_actions": { "n": 41, "mean": 5.121951219512195, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 41, "mean": 193.25609329694697, "median": 187.0, "p90": 237.0 }, "generated_tokens": { "n": 41, "mean": 1033.0487804878048, "median": 793.0, "p90": 2040.0 }, "first_turn_tokens": { "n": 41, "mean": 151.0731707317073, "median": 160.0, "p90": 206.0 }, "searches": { "n": 41, "mean": 1.3658536585365855, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 41, "mean": 1.3658536585365855, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 41, "mean": 1.2682926829268293, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 41, "mean": 1.170731707317073, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 41, "mean": 0.0975609756097561, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 41, "mean": 0.36585365853658536, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 41, "mean": 0.04878048780487805, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 41, "mean": 1.2195121951219512, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 41, "mean": 0.07317073170731707, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 41, "mean": 0.04878048780487805, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 41, "mean": 0.024390243902439025, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 41, "mean": 246.23522940474157, "median": 234.75, "p90": 308.72727272727275 }, "first_product_rank": { "n": 41, "mean": 1.975609756097561, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 193.40182648401827, "token_complete_episodes": 41, "rates": { "protocol_errors": 0.04878048780487805, "product_revisits": 0.0975609756097561, "option_changes": 0.04878048780487805, "backtracks": 0.1951219512195122 }, "terminations": { "generation_length": 6, "purchase": 35 } } }, { "step": 58, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.01953125, "median": 4.0, "p90": 8.0 }, "actions": { "n": 256, "mean": 4.98046875, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 4.98046875, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 187.27577719656415, "median": 178.625, "p90": 238.95833333333331 }, "generated_tokens": { "n": 256, "mean": 962.6015625, "median": 755.5, "p90": 1658.5 }, "first_turn_tokens": { "n": 256, "mean": 145.5625, "median": 137.0, "p90": 202.5 }, "searches": { "n": 256, "mean": 1.21484375, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.20703125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.20703125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.08984375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.1171875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.21484375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.34765625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.09765625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 235.7527834160773, "median": 224.75, "p90": 316.05 }, "first_product_rank": { "n": 256, "mean": 1.22265625, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.8984375, "reward_mean": 0.8984375, "pooled_tokens_per_turn": 191.77120622568094, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.03515625, "product_revisits": 0.08984375, "option_changes": 0.0390625, "backtracks": 0.1328125 }, "terminations": { "purchase": 255, "generation_length": 1 } }, "success": { "n": 230, "tasks": 30, "metrics": { "model_steps": { "n": 230, "mean": 4.582608695652174, "median": 4.0, "p90": 5.0 }, "actions": { "n": 230, "mean": 4.556521739130435, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 230, "mean": 4.556521739130435, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 230, "mean": 181.2079351928265, "median": 176.5, "p90": 225.29999999999998 }, "generated_tokens": { "n": 230, "mean": 838.8869565217391, "median": 732.5, "p90": 1238.8 }, "first_turn_tokens": { "n": 230, "mean": 142.6782608695652, "median": 136.0, "p90": 189.1 }, "searches": { "n": 230, "mean": 1.117391304347826, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 230, "mean": 1.108695652173913, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 230, "mean": 1.1130434782608696, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 230, "mean": 1.0391304347826087, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 230, "mean": 0.07391304347826087, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 230, "mean": 0.11739130434782609, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 230, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 230, "mean": 1.208695652173913, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 230, "mean": 0.034782608695652174, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 230, "mean": 0.02608695652173913, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 230, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 230, "mean": 226.71214064920585, "median": 221.5, "p90": 295.075 }, "first_product_rank": { "n": 230, "mean": 1.1956521739130435, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 183.05882352941177, "token_complete_episodes": 230, "rates": { "protocol_errors": 0.02608695652173913, "product_revisits": 0.06956521739130435, "option_changes": 0.030434782608695653, "backtracks": 0.09130434782608696 }, "terminations": { "purchase": 230 } }, "failure": { "n": 26, "tasks": 6, "metrics": { "model_steps": { "n": 26, "mean": 8.884615384615385, "median": 8.0, "p90": 17.0 }, "actions": { "n": 26, "mean": 8.73076923076923, "median": 8.0, "p90": 16.5 }, "valid_actions": { "n": 26, "mean": 8.73076923076923, "median": 8.0, "p90": 16.5 }, "tokens_per_turn": { "n": 26, "mean": 240.9528410757816, "median": 231.85833333333335, "p90": 295.40625 }, "generated_tokens": { "n": 26, "mean": 2057.0, "median": 1711.0, "p90": 3980.0 }, "first_turn_tokens": { "n": 26, "mean": 171.07692307692307, "median": 159.5, "p90": 236.5 }, "searches": { "n": 26, "mean": 2.076923076923077, "median": 1.5, "p90": 4.0 }, "unique_queries": { "n": 26, "mean": 2.076923076923077, "median": 1.5, "p90": 4.0 }, "product_opens": { "n": 26, "mean": 2.0384615384615383, "median": 1.5, "p90": 4.0 }, "unique_products": { "n": 26, "mean": 1.5384615384615385, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 26, "mean": 0.5, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 26, "mean": 1.0769230769230769, "median": 0.5, "p90": 3.0 }, "pagination": { "n": 26, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 26, "mean": 2.576923076923077, "median": 2.0, "p90": 4.5 }, "option_changes": { "n": 26, "mean": 0.6538461538461539, "median": 0.0, "p90": 0.5 }, "protocol_errors": { "n": 26, "mean": 0.11538461538461539, "median": 0.0, "p90": 0.5 }, "invalid_actions": { "n": 26, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 26, "mean": 315.7277001999398, "median": 303.31666666666666, "p90": 401.59375 }, "first_product_rank": { "n": 26, "mean": 1.4615384615384615, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 231.52380952380952, "token_complete_episodes": 26, "rates": { "protocol_errors": 0.11538461538461539, "product_revisits": 0.2692307692307692, "option_changes": 0.11538461538461539, "backtracks": 0.5 }, "terminations": { "purchase": 25, "generation_length": 1 } } }, { "step": 59, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.30859375, "median": 4.0, "p90": 6.0 }, "actions": { "n": 256, "mean": 5.22265625, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 256, "mean": 5.20703125, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 256, "mean": 181.3095533064648, "median": 176.0, "p90": 234.875 }, "generated_tokens": { "n": 256, "mean": 954.9921875, "median": 768.0, "p90": 1258.5 }, "first_turn_tokens": { "n": 256, "mean": 136.73828125, "median": 129.0, "p90": 191.5 }, "searches": { "n": 256, "mean": 1.18359375, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.1796875, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.1484375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.08203125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.06640625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.18359375, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.71484375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.28125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 227.31115933592753, "median": 214.875, "p90": 309.0 }, "first_product_rank": { "n": 256, "mean": 1.10546875, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.8671875, "reward_mean": 0.8683035714285714, "pooled_tokens_per_turn": 179.89551140544518, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.04296875, "product_revisits": 0.05078125, "option_changes": 0.06640625, "backtracks": 0.0859375 }, "terminations": { "purchase": 247, "generation_length": 9 } }, "success": { "n": 222, "tasks": 30, "metrics": { "model_steps": { "n": 222, "mean": 4.603603603603603, "median": 4.0, "p90": 5.0 }, "actions": { "n": 222, "mean": 4.576576576576577, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 222, "mean": 4.563063063063063, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 222, "mean": 181.27704930204928, "median": 177.1, "p90": 233.5 }, "generated_tokens": { "n": 222, "mean": 843.8153153153153, "median": 756.0, "p90": 1026.3 }, "first_turn_tokens": { "n": 222, "mean": 139.82882882882882, "median": 132.0, "p90": 198.60000000000002 }, "searches": { "n": 222, "mean": 1.0675675675675675, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 222, "mean": 1.063063063063063, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 222, "mean": 1.0585585585585586, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 222, "mean": 1.0405405405405406, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 222, "mean": 0.018018018018018018, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 222, "mean": 0.06756756756756757, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 222, "mean": 0.0045045045045045045, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 222, "mean": 1.364864864864865, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 222, "mean": 0.06306306306306306, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 222, "mean": 0.02702702702702703, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 222, "mean": 0.013513513513513514, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 222, "mean": 226.33300347050346, "median": 216.25, "p90": 309.2 }, "first_product_rank": { "n": 222, "mean": 1.0990990990990992, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 183.2945205479452, "token_complete_episodes": 222, "rates": { "protocol_errors": 0.02252252252252252, "product_revisits": 0.018018018018018018, "option_changes": 0.03153153153153153, "backtracks": 0.04954954954954955 }, "terminations": { "purchase": 222 } }, "failure": { "n": 34, "tasks": 8, "metrics": { "model_steps": { "n": 34, "mean": 9.911764705882353, "median": 4.0, "p90": 24.7 }, "actions": { "n": 34, "mean": 9.441176470588236, "median": 4.0, "p90": 22.7 }, "valid_actions": { "n": 34, "mean": 9.411764705882353, "median": 4.0, "p90": 22.7 }, "tokens_per_turn": { "n": 34, "mean": 181.52178533529556, "median": 160.6875, "p90": 235.425 }, "generated_tokens": { "n": 34, "mean": 1680.9117647058824, "median": 874.0, "p90": 3849.9 }, "first_turn_tokens": { "n": 34, "mean": 116.55882352941177, "median": 114.5, "p90": 147.1 }, "searches": { "n": 34, "mean": 1.9411764705882353, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 34, "mean": 1.9411764705882353, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 34, "mean": 1.7352941176470589, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 34, "mean": 1.3529411764705883, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 34, "mean": 0.38235294117647056, "median": 0.0, "p90": 1.6999999999999993 }, "backtracks": { "n": 34, "mean": 0.9411764705882353, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 34, "mean": 0.058823529411764705, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 34, "mean": 4.0, "median": 1.0, "p90": 13.0 }, "option_changes": { "n": 34, "mean": 1.7058823529411764, "median": 0.0, "p90": 7.0 }, "protocol_errors": { "n": 34, "mean": 0.20588235294117646, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 34, "mean": 0.029411764705882353, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 34, "mean": 233.69794175134325, "median": 201.78571428571428, "p90": 305.05760869565216 }, "first_product_rank": { "n": 34, "mean": 1.1470588235294117, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.008403361344537815, "pooled_tokens_per_turn": 169.58753709198814, "token_complete_episodes": 34, "rates": { "protocol_errors": 0.17647058823529413, "product_revisits": 0.2647058823529412, "option_changes": 0.29411764705882354, "backtracks": 0.3235294117647059 }, "terminations": { "generation_length": 9, "purchase": 25 } } }, { "step": 60, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.0703125, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.984375, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.92578125, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 198.86948699233358, "median": 194.75, "p90": 256.875 }, "generated_tokens": { "n": 256, "mean": 1003.53515625, "median": 828.0, "p90": 1330.0 }, "first_turn_tokens": { "n": 256, "mean": 148.5, "median": 143.5, "p90": 204.5 }, "searches": { "n": 256, "mean": 1.203125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.19140625, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.1484375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.09765625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.203125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.06640625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.34375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.05859375, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 252.20500694848437, "median": 243.2, "p90": 332.125 }, "first_product_rank": { "n": 256, "mean": 1.34765625, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.796875, "reward_mean": 0.8460286458333333, "pooled_tokens_per_turn": 197.92372881355934, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0390625, "product_revisits": 0.0390625, "option_changes": 0.00390625, "backtracks": 0.08984375 }, "terminations": { "purchase": 246, "generation_length": 10 } }, "success": { "n": 204, "tasks": 28, "metrics": { "model_steps": { "n": 204, "mean": 4.3088235294117645, "median": 4.0, "p90": 5.0 }, "actions": { "n": 204, "mean": 4.3088235294117645, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 204, "mean": 4.3088235294117645, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 204, "mean": 196.08582127295364, "median": 188.25, "p90": 253.82500000000005 }, "generated_tokens": { "n": 204, "mean": 846.3284313725491, "median": 792.5, "p90": 1089.1000000000004 }, "first_turn_tokens": { "n": 204, "mean": 148.62254901960785, "median": 143.0, "p90": 206.0 }, "searches": { "n": 204, "mean": 1.0392156862745099, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 204, "mean": 1.0294117647058822, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 204, "mean": 1.0343137254901962, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 204, "mean": 1.0294117647058822, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 204, "mean": 0.004901960784313725, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 204, "mean": 0.0392156862745098, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 204, "mean": 0.00980392156862745, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 204, "mean": 1.1862745098039216, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 204, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 204, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 204, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 204, "mean": 247.03448879551823, "median": 235.55, "p90": 326.7500000000001 }, "first_product_rank": { "n": 204, "mean": 1.2598039215686274, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 196.4175199089875, "token_complete_episodes": 204, "rates": { "protocol_errors": 0.0, "product_revisits": 0.004901960784313725, "option_changes": 0.0, "backtracks": 0.03431372549019608 }, "terminations": { "purchase": 204 } }, "failure": { "n": 52, "tasks": 10, "metrics": { "model_steps": { "n": 52, "mean": 8.057692307692308, "median": 5.0, "p90": 18.0 }, "actions": { "n": 52, "mean": 7.634615384615385, "median": 5.0, "p90": 16.9 }, "valid_actions": { "n": 52, "mean": 7.346153846153846, "median": 5.0, "p90": 15.899999999999999 }, "tokens_per_turn": { "n": 52, "mean": 209.79002173759335, "median": 204.25735294117646, "p90": 260.815 }, "generated_tokens": { "n": 52, "mean": 1620.2692307692307, "median": 1123.5, "p90": 3531.5 }, "first_turn_tokens": { "n": 52, "mean": 148.01923076923077, "median": 146.5, "p90": 188.9 }, "searches": { "n": 52, "mean": 1.8461538461538463, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 52, "mean": 1.8269230769230769, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 52, "mean": 1.5961538461538463, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 52, "mean": 1.3653846153846154, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 52, "mean": 0.23076923076923078, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 52, "mean": 0.8461538461538461, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 52, "mean": 0.28846153846153844, "median": 0.0, "p90": 1.8999999999999986 }, "option_selections": { "n": 52, "mean": 1.9615384615384615, "median": 2.0, "p90": 3.8999999999999986 }, "option_changes": { "n": 52, "mean": 0.019230769230769232, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 52, "mean": 0.23076923076923078, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 52, "mean": 0.28846153846153844, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 52, "mean": 272.4893473947363, "median": 261.0272727272727, "p90": 344.46999999999997 }, "first_product_rank": { "n": 52, "mean": 1.6923076923076923, "median": 1.0, "p90": 2.799999999999997 } }, "success_rate": 0.0, "reward_mean": 0.24198717948717946, "pooled_tokens_per_turn": 201.0835322195704, "token_complete_episodes": 52, "rates": { "protocol_errors": 0.19230769230769232, "product_revisits": 0.17307692307692307, "option_changes": 0.019230769230769232, "backtracks": 0.3076923076923077 }, "terminations": { "purchase": 42, "generation_length": 10 } } }, { "step": 61, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.61328125, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.55859375, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.5390625, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 220.84937338146833, "median": 209.3, "p90": 283.125 }, "generated_tokens": { "n": 256, "mean": 1028.73046875, "median": 877.0, "p90": 1381.5 }, "first_turn_tokens": { "n": 256, "mean": 180.86328125, "median": 174.5, "p90": 239.0 }, "searches": { "n": 256, "mean": 1.10546875, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.09375, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.09375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0390625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.0546875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.109375, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.234375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 280.78010519730117, "median": 268.125, "p90": 378.125 }, "first_product_rank": { "n": 256, "mean": 1.16796875, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.87109375, "reward_mean": 0.871875, "pooled_tokens_per_turn": 222.99322607959357, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.02734375, "product_revisits": 0.04296875, "option_changes": 0.03125, "backtracks": 0.07421875 }, "terminations": { "purchase": 253, "generation_length": 3 } }, "success": { "n": 223, "tasks": 30, "metrics": { "model_steps": { "n": 223, "mean": 4.466367713004484, "median": 4.0, "p90": 5.0 }, "actions": { "n": 223, "mean": 4.452914798206278, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 223, "mean": 4.434977578475336, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 223, "mean": 211.84057146142348, "median": 203.75, "p90": 269.65000000000003 }, "generated_tokens": { "n": 223, "mean": 943.5291479820628, "median": 851.0, "p90": 1281.4 }, "first_turn_tokens": { "n": 223, "mean": 181.05829596412556, "median": 174.0, "p90": 238.0 }, "searches": { "n": 223, "mean": 1.0717488789237668, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 223, "mean": 1.062780269058296, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 223, "mean": 1.0672645739910314, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 223, "mean": 1.0358744394618835, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 223, "mean": 0.03139013452914798, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 223, "mean": 0.07174887892376682, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 223, "mean": 0.004484304932735426, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 223, "mean": 1.2197309417040358, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 223, "mean": 0.03139013452914798, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 223, "mean": 0.013452914798206279, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 223, "mean": 0.017937219730941704, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 223, "mean": 273.69558468437396, "median": 265.5, "p90": 362.3900000000001 }, "first_product_rank": { "n": 223, "mean": 1.1255605381165918, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 211.25200803212851, "token_complete_episodes": 223, "rates": { "protocol_errors": 0.013452914798206279, "product_revisits": 0.03139013452914798, "option_changes": 0.03139013452914798, "backtracks": 0.06278026905829596 }, "terminations": { "purchase": 223 } }, "failure": { "n": 33, "tasks": 10, "metrics": { "model_steps": { "n": 33, "mean": 5.606060606060606, "median": 4.0, "p90": 9.600000000000001 }, "actions": { "n": 33, "mean": 5.2727272727272725, "median": 4.0, "p90": 8.8 }, "valid_actions": { "n": 33, "mean": 5.242424242424242, "median": 4.0, "p90": 8.8 }, "tokens_per_turn": { "n": 33, "mean": 281.7270348411653, "median": 253.0, "p90": 347.0444444444444 }, "generated_tokens": { "n": 33, "mean": 1604.4848484848485, "median": 1058.0, "p90": 3893.8 }, "first_turn_tokens": { "n": 33, "mean": 179.54545454545453, "median": 182.0, "p90": 246.0 }, "searches": { "n": 33, "mean": 1.3333333333333333, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 33, "mean": 1.303030303030303, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 33, "mean": 1.2727272727272727, "median": 1.0, "p90": 1.8000000000000007 }, "unique_products": { "n": 33, "mean": 1.0606060606060606, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 33, "mean": 0.21212121212121213, "median": 0.0, "p90": 0.8000000000000007 }, "backtracks": { "n": 33, "mean": 0.36363636363636365, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 33, "mean": 0.030303030303030304, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 33, "mean": 1.3333333333333333, "median": 1.0, "p90": 2.8000000000000007 }, "option_changes": { "n": 33, "mean": 0.030303030303030304, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 33, "mean": 0.24242424242424243, "median": 0.0, "p90": 0.8000000000000007 }, "invalid_actions": { "n": 33, "mean": 0.030303030303030304, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 33, "mean": 328.65428926950665, "median": 336.25, "p90": 473.85 }, "first_product_rank": { "n": 33, "mean": 1.4545454545454546, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.0, "reward_mean": 0.006060606060606061, "pooled_tokens_per_turn": 286.20540540540543, "token_complete_episodes": 33, "rates": { "protocol_errors": 0.12121212121212122, "product_revisits": 0.12121212121212122, "option_changes": 0.030303030303030304, "backtracks": 0.15151515151515152 }, "terminations": { "purchase": 30, "generation_length": 3 } } }, { "step": 62, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.09375, "median": 4.0, "p90": 8.0 }, "actions": { "n": 256, "mean": 5.01953125, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.01171875, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 224.40932183084885, "median": 215.75, "p90": 285.625 }, "generated_tokens": { "n": 256, "mean": 1141.7578125, "median": 911.5, "p90": 1849.0 }, "first_turn_tokens": { "n": 256, "mean": 184.13671875, "median": 186.0, "p90": 245.5 }, "searches": { "n": 256, "mean": 1.2109375, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.20703125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.18359375, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.08984375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.09375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.2109375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.40625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.05859375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 291.40744974181627, "median": 284.875, "p90": 381.5 }, "first_product_rank": { "n": 256, "mean": 1.390625, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.8359375, "reward_mean": 0.8563802083333334, "pooled_tokens_per_turn": 224.14877300613497, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.05859375, "product_revisits": 0.07421875, "option_changes": 0.015625, "backtracks": 0.1171875 }, "terminations": { "purchase": 252, "generation_length": 4 } }, "success": { "n": 214, "tasks": 28, "metrics": { "model_steps": { "n": 214, "mean": 4.4672897196261685, "median": 4.0, "p90": 5.0 }, "actions": { "n": 214, "mean": 4.420560747663552, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 214, "mean": 4.420560747663552, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 214, "mean": 218.92819188517663, "median": 213.25, "p90": 285.975 }, "generated_tokens": { "n": 214, "mean": 976.8364485981308, "median": 880.5, "p90": 1261.7 }, "first_turn_tokens": { "n": 214, "mean": 182.15420560747663, "median": 182.5, "p90": 239.10000000000005 }, "searches": { "n": 214, "mean": 1.0841121495327102, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 214, "mean": 1.0841121495327102, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 214, "mean": 1.0794392523364487, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 214, "mean": 1.0233644859813085, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 214, "mean": 0.056074766355140186, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 214, "mean": 0.08411214953271028, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 214, "mean": 0.004672897196261682, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 214, "mean": 1.1682242990654206, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 214, "mean": 0.018691588785046728, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 214, "mean": 0.04672897196261682, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 214, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 214, "mean": 288.0618663841487, "median": 280.275, "p90": 381.6500000000001 }, "first_product_rank": { "n": 214, "mean": 1.3691588785046729, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 218.6642259414226, "token_complete_episodes": 214, "rates": { "protocol_errors": 0.04672897196261682, "product_revisits": 0.0514018691588785, "option_changes": 0.018691588785046728, "backtracks": 0.056074766355140186 }, "terminations": { "purchase": 214 } }, "failure": { "n": 42, "tasks": 10, "metrics": { "model_steps": { "n": 42, "mean": 8.285714285714286, "median": 5.0, "p90": 16.799999999999997 }, "actions": { "n": 42, "mean": 8.071428571428571, "median": 5.0, "p90": 15.899999999999999 }, "valid_actions": { "n": 42, "mean": 8.023809523809524, "median": 5.0, "p90": 15.899999999999999 }, "tokens_per_turn": { "n": 42, "mean": 252.33698393498813, "median": 226.25, "p90": 272.3857142857143 }, "generated_tokens": { "n": 42, "mean": 1982.0714285714287, "median": 1471.5, "p90": 3686.1 }, "first_turn_tokens": { "n": 42, "mean": 194.23809523809524, "median": 190.5, "p90": 254.5 }, "searches": { "n": 42, "mean": 1.8571428571428572, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 42, "mean": 1.8333333333333333, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 42, "mean": 1.7142857142857142, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 42, "mean": 1.4285714285714286, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 42, "mean": 0.2857142857142857, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 42, "mean": 0.8571428571428571, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 42, "mean": 0.07142857142857142, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 42, "mean": 2.619047619047619, "median": 2.0, "p90": 5.0 }, "option_changes": { "n": 42, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 42, "mean": 0.11904761904761904, "median": 0.0, "p90": 0.8999999999999986 }, "invalid_actions": { "n": 42, "mean": 0.047619047619047616, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 42, "mean": 308.4539935165986, "median": 305.3, "p90": 379.35714285714283 }, "first_product_rank": { "n": 42, "mean": 1.5, "median": 1.0, "p90": 2.8999999999999986 } }, "success_rate": 0.0, "reward_mean": 0.12460317460317458, "pooled_tokens_per_turn": 239.2155172413793, "token_complete_episodes": 42, "rates": { "protocol_errors": 0.11904761904761904, "product_revisits": 0.19047619047619047, "option_changes": 0.0, "backtracks": 0.42857142857142855 }, "terminations": { "purchase": 38, "generation_length": 4 } } }, { "step": 63, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.8046875, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.734375, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.73046875, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 237.84954336028557, "median": 226.5, "p90": 298.0 }, "generated_tokens": { "n": 256, "mean": 1164.296875, "median": 954.5, "p90": 1636.0 }, "first_turn_tokens": { "n": 256, "mean": 200.3984375, "median": 199.5, "p90": 257.5 }, "searches": { "n": 256, "mean": 1.11328125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.10546875, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.09375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.05078125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.1171875, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.40625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.05859375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0546875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 314.36543466299327, "median": 301.625, "p90": 413.125 }, "first_product_rank": { "n": 256, "mean": 1.21875, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.87890625, "reward_mean": 0.8984375, "pooled_tokens_per_turn": 242.3252032520325, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.04296875, "product_revisits": 0.0390625, "option_changes": 0.03515625, "backtracks": 0.08203125 }, "terminations": { "purchase": 252, "generation_length": 4 } }, "success": { "n": 225, "tasks": 30, "metrics": { "model_steps": { "n": 225, "mean": 4.493333333333333, "median": 4.0, "p90": 5.0 }, "actions": { "n": 225, "mean": 4.471111111111111, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 225, "mean": 4.466666666666667, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 225, "mean": 227.3704748338082, "median": 221.2, "p90": 284.94999999999993 }, "generated_tokens": { "n": 225, "mean": 1025.3466666666666, "median": 930.0, "p90": 1312.1999999999998 }, "first_turn_tokens": { "n": 225, "mean": 197.1822222222222, "median": 197.0, "p90": 252.0 }, "searches": { "n": 225, "mean": 1.0577777777777777, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 225, "mean": 1.0577777777777777, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 225, "mean": 1.048888888888889, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 225, "mean": 1.0133333333333334, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 225, "mean": 0.035555555555555556, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 225, "mean": 0.057777777777777775, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 225, "mean": 0.008888888888888889, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 225, "mean": 1.2933333333333332, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 225, "mean": 0.04, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 225, "mean": 0.022222222222222223, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 225, "mean": 0.0044444444444444444, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 225, "mean": 301.50648826482154, "median": 294.5, "p90": 395.8 }, "first_product_rank": { "n": 225, "mean": 1.1288888888888888, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 228.19287833827894, "token_complete_episodes": 225, "rates": { "protocol_errors": 0.022222222222222223, "product_revisits": 0.03111111111111111, "option_changes": 0.035555555555555556, "backtracks": 0.05333333333333334 }, "terminations": { "purchase": 225 } }, "failure": { "n": 31, "tasks": 8, "metrics": { "model_steps": { "n": 31, "mean": 7.064516129032258, "median": 5.0, "p90": 11.0 }, "actions": { "n": 31, "mean": 6.645161290322581, "median": 5.0, "p90": 10.0 }, "valid_actions": { "n": 31, "mean": 6.645161290322581, "median": 5.0, "p90": 10.0 }, "tokens_per_turn": { "n": 31, "mean": 313.9072987943956, "median": 263.4, "p90": 408.90909090909093 }, "generated_tokens": { "n": 31, "mean": 2172.8064516129034, "median": 1440.0, "p90": 4541.0 }, "first_turn_tokens": { "n": 31, "mean": 223.74193548387098, "median": 216.0, "p90": 280.0 }, "searches": { "n": 31, "mean": 1.5161290322580645, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 31, "mean": 1.4516129032258065, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 31, "mean": 1.4193548387096775, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 31, "mean": 1.3225806451612903, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 31, "mean": 0.0967741935483871, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 31, "mean": 0.5483870967741935, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 31, "mean": 0.06451612903225806, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 31, "mean": 2.225806451612903, "median": 2.0, "p90": 4.0 }, "option_changes": { "n": 31, "mean": 0.1935483870967742, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 31, "mean": 0.2903225806451613, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 31, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 31, "mean": 407.6964972303682, "median": 358.6, "p90": 586.25 }, "first_product_rank": { "n": 31, "mean": 1.8709677419354838, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.0, "reward_mean": 0.16129032258064516, "pooled_tokens_per_turn": 307.5662100456621, "token_complete_episodes": 31, "rates": { "protocol_errors": 0.1935483870967742, "product_revisits": 0.0967741935483871, "option_changes": 0.03225806451612903, "backtracks": 0.2903225806451613 }, "terminations": { "purchase": 27, "generation_length": 4 } } }, { "step": 64, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.4296875, "median": 4.0, "p90": 10.0 }, "actions": { "n": 256, "mean": 5.265625, "median": 4.0, "p90": 9.0 }, "valid_actions": { "n": 256, "mean": 5.13671875, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 228.25854310905197, "median": 216.125, "p90": 295.25 }, "generated_tokens": { "n": 256, "mean": 1273.8515625, "median": 954.5, "p90": 2391.0 }, "first_turn_tokens": { "n": 256, "mean": 187.42578125, "median": 189.5, "p90": 242.0 }, "searches": { "n": 256, "mean": 1.2421875, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.2421875, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.12890625, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.05078125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.2421875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.23828125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.35546875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.07421875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.09375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.12890625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 303.39377912241036, "median": 288.925, "p90": 409.0 }, "first_product_rank": { "n": 248, "mean": 1.310483870967742, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.8359375, "reward_mean": 0.8587239583333334, "pooled_tokens_per_turn": 234.60863309352519, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.08203125, "product_revisits": 0.06640625, "option_changes": 0.04296875, "backtracks": 0.16796875 }, "terminations": { "generation_length": 18, "purchase": 238 } }, "success": { "n": 214, "tasks": 31, "metrics": { "model_steps": { "n": 214, "mean": 4.822429906542056, "median": 4.0, "p90": 6.700000000000017 }, "actions": { "n": 214, "mean": 4.780373831775701, "median": 4.0, "p90": 6.700000000000017 }, "valid_actions": { "n": 214, "mean": 4.780373831775701, "median": 4.0, "p90": 6.700000000000017 }, "tokens_per_turn": { "n": 214, "mean": 221.70471693415618, "median": 214.0, "p90": 285.94 }, "generated_tokens": { "n": 214, "mean": 1075.7056074766356, "median": 923.0, "p90": 1701.1000000000022 }, "first_turn_tokens": { "n": 214, "mean": 185.15887850467288, "median": 188.0, "p90": 239.0 }, "searches": { "n": 214, "mean": 1.1261682242990654, "median": 1.0, "p90": 1.700000000000017 }, "unique_queries": { "n": 214, "mean": 1.1261682242990654, "median": 1.0, "p90": 1.700000000000017 }, "product_opens": { "n": 214, "mean": 1.1121495327102804, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 214, "mean": 1.074766355140187, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 214, "mean": 0.037383177570093455, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 214, "mean": 0.1261682242990654, "median": 0.0, "p90": 0.700000000000017 }, "pagination": { "n": 214, "mean": 0.028037383177570093, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 214, "mean": 1.3878504672897196, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 214, "mean": 0.08411214953271028, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 214, "mean": 0.04205607476635514, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 214, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 214, "mean": 293.3992750175577, "median": 284.25, "p90": 382.8 }, "first_product_rank": { "n": 214, "mean": 1.2570093457943925, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 223.06298449612405, "token_complete_episodes": 214, "rates": { "protocol_errors": 0.04205607476635514, "product_revisits": 0.03271028037383177, "option_changes": 0.04672897196261682, "backtracks": 0.102803738317757 }, "terminations": { "purchase": 214 } }, "failure": { "n": 42, "tasks": 11, "metrics": { "model_steps": { "n": 42, "mean": 8.523809523809524, "median": 8.5, "p90": 12.899999999999999 }, "actions": { "n": 42, "mean": 7.738095238095238, "median": 8.0, "p90": 11.0 }, "valid_actions": { "n": 42, "mean": 6.9523809523809526, "median": 6.0, "p90": 10.0 }, "tokens_per_turn": { "n": 42, "mean": 261.6518479049494, "median": 225.0, "p90": 402.0381818181818 }, "generated_tokens": { "n": 42, "mean": 2283.4523809523807, "median": 2050.5, "p90": 4140.599999999999 }, "first_turn_tokens": { "n": 42, "mean": 198.97619047619048, "median": 198.0, "p90": 254.7 }, "searches": { "n": 42, "mean": 1.8333333333333333, "median": 1.5, "p90": 3.0 }, "unique_queries": { "n": 42, "mean": 1.8333333333333333, "median": 1.5, "p90": 3.0 }, "product_opens": { "n": 42, "mean": 1.2142857142857142, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 42, "mean": 0.9285714285714286, "median": 1.0, "p90": 1.8999999999999986 }, "product_revisits": { "n": 42, "mean": 0.2857142857142857, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 42, "mean": 0.8333333333333334, "median": 0.5, "p90": 2.0 }, "pagination": { "n": 42, "mean": 1.3095238095238095, "median": 0.0, "p90": 7.0 }, "option_selections": { "n": 42, "mean": 1.1904761904761905, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 42, "mean": 0.023809523809523808, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 42, "mean": 0.35714285714285715, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 42, "mean": 0.7857142857142857, "median": 0.0, "p90": 3.0 }, "reasoning_chars_per_turn": { "n": 42, "mean": 354.31815718046926, "median": 310.6285714285714, "p90": 529.2579545454545 }, "first_product_rank": { "n": 34, "mean": 1.6470588235294117, "median": 1.0, "p90": 3.6999999999999993 } }, "success_rate": 0.0, "reward_mean": 0.1388888888888889, "pooled_tokens_per_turn": 267.89106145251395, "token_complete_episodes": 42, "rates": { "protocol_errors": 0.2857142857142857, "product_revisits": 0.23809523809523808, "option_changes": 0.023809523809523808, "backtracks": 0.5 }, "terminations": { "generation_length": 18, "purchase": 24 } } }, { "step": 65, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.59375, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.5546875, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.54296875, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 224.01190941220236, "median": 212.25, "p90": 292.625 }, "generated_tokens": { "n": 256, "mean": 1036.0390625, "median": 890.0, "p90": 1442.0 }, "first_turn_tokens": { "n": 256, "mean": 185.16796875, "median": 189.0, "p90": 241.5 }, "searches": { "n": 256, "mean": 1.09375, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.09375, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.0625, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0234375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.0390625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.09375, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.05859375, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.2421875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 297.41527169661936, "median": 284.1, "p90": 387.25 }, "first_product_rank": { "n": 254, "mean": 1.1496062992125984, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.83984375, "reward_mean": 0.83984375, "pooled_tokens_per_turn": 225.53231292517006, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.03125, "product_revisits": 0.0390625, "option_changes": 0.01171875, "backtracks": 0.06640625 }, "terminations": { "purchase": 254, "generation_length": 2 } }, "success": { "n": 215, "tasks": 28, "metrics": { "model_steps": { "n": 215, "mean": 4.534883720930233, "median": 4.0, "p90": 5.0 }, "actions": { "n": 215, "mean": 4.502325581395349, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 215, "mean": 4.493023255813953, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 215, "mean": 216.6636165772212, "median": 210.2, "p90": 271.09999999999997 }, "generated_tokens": { "n": 215, "mean": 993.6093023255814, "median": 876.0, "p90": 1270.0 }, "first_turn_tokens": { "n": 215, "mean": 187.05581395348838, "median": 191.0, "p90": 242.6 }, "searches": { "n": 215, "mean": 1.0744186046511628, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 215, "mean": 1.0744186046511628, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 215, "mean": 1.0651162790697675, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 215, "mean": 1.0325581395348837, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 215, "mean": 0.03255813953488372, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 215, "mean": 0.07441860465116279, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 215, "mean": 0.018604651162790697, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 215, "mean": 1.2604651162790699, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 215, "mean": 0.004651162790697674, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 215, "mean": 0.03255813953488372, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 215, "mean": 0.009302325581395349, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 215, "mean": 285.105322997416, "median": 278.5, "p90": 372.1 }, "first_product_rank": { "n": 215, "mean": 1.083720930232558, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 219.10358974358974, "token_complete_episodes": 215, "rates": { "protocol_errors": 0.03255813953488372, "product_revisits": 0.03255813953488372, "option_changes": 0.004651162790697674, "backtracks": 0.05116279069767442 }, "terminations": { "purchase": 215 } }, "failure": { "n": 41, "tasks": 8, "metrics": { "model_steps": { "n": 41, "mean": 4.902439024390244, "median": 4.0, "p90": 8.0 }, "actions": { "n": 41, "mean": 4.829268292682927, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 41, "mean": 4.804878048780488, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 41, "mean": 262.5456401322255, "median": 261.85714285714283, "p90": 351.0 }, "generated_tokens": { "n": 41, "mean": 1258.5365853658536, "median": 1139.0, "p90": 2049.0 }, "first_turn_tokens": { "n": 41, "mean": 175.26829268292684, "median": 161.0, "p90": 229.0 }, "searches": { "n": 41, "mean": 1.1951219512195121, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 41, "mean": 1.1951219512195121, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 41, "mean": 1.048780487804878, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 41, "mean": 0.975609756097561, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 41, "mean": 0.07317073170731707, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 41, "mean": 0.1951219512195122, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 41, "mean": 0.2682926829268293, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 41, "mean": 1.146341463414634, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 41, "mean": 0.0975609756097561, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 41, "mean": 0.024390243902439025, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 41, "mean": 0.024390243902439025, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 41, "mean": 361.9674417046368, "median": 359.5, "p90": 506.0 }, "first_product_rank": { "n": 39, "mean": 1.5128205128205128, "median": 1.0, "p90": 1.2000000000000028 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 256.7164179104478, "token_complete_episodes": 41, "rates": { "protocol_errors": 0.024390243902439025, "product_revisits": 0.07317073170731707, "option_changes": 0.04878048780487805, "backtracks": 0.14634146341463414 }, "terminations": { "purchase": 39, "generation_length": 2 } } }, { "step": 66, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.12890625, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 5.02734375, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.74609375, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 228.29776467794437, "median": 217.325, "p90": 292.65 }, "generated_tokens": { "n": 256, "mean": 1216.48828125, "median": 914.5, "p90": 1716.0 }, "first_turn_tokens": { "n": 256, "mean": 180.83984375, "median": 177.5, "p90": 252.5 }, "searches": { "n": 256, "mean": 1.2109375, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.2109375, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.171875, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0390625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.1328125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.2109375, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.1796875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0546875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.28125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 298.944657097641, "median": 285.25, "p90": 397.5 }, "first_product_rank": { "n": 256, "mean": 1.36328125, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.859375, "reward_mean": 0.8621651785714286, "pooled_tokens_per_turn": 237.18278750952018, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.05078125, "product_revisits": 0.06640625, "option_changes": 0.015625, "backtracks": 0.0859375 }, "terminations": { "purchase": 244, "generation_length": 12 } }, "success": { "n": 220, "tasks": 29, "metrics": { "model_steps": { "n": 220, "mean": 4.327272727272727, "median": 4.0, "p90": 5.0 }, "actions": { "n": 220, "mean": 4.290909090909091, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 220, "mean": 4.286363636363636, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 220, "mean": 217.88709054834055, "median": 214.125, "p90": 274.425 }, "generated_tokens": { "n": 220, "mean": 941.5454545454545, "median": 891.0, "p90": 1240.1 }, "first_turn_tokens": { "n": 220, "mean": 181.90454545454546, "median": 177.0, "p90": 254.0 }, "searches": { "n": 220, "mean": 1.0318181818181817, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 220, "mean": 1.0318181818181817, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 220, "mean": 1.018181818181818, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 220, "mean": 1.0136363636363637, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 220, "mean": 0.004545454545454545, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 220, "mean": 0.031818181818181815, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 220, "mean": 0.013636363636363636, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 220, "mean": 1.190909090909091, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 220, "mean": 0.004545454545454545, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 220, "mean": 0.03636363636363636, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 220, "mean": 0.004545454545454545, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 220, "mean": 284.6009163059163, "median": 281.0, "p90": 369.54999999999995 }, "first_product_rank": { "n": 220, "mean": 1.4090909090909092, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 217.58403361344537, "token_complete_episodes": 220, "rates": { "protocol_errors": 0.03636363636363636, "product_revisits": 0.004545454545454545, "option_changes": 0.004545454545454545, "backtracks": 0.02727272727272727 }, "terminations": { "purchase": 220 } }, "failure": { "n": 36, "tasks": 7, "metrics": { "model_steps": { "n": 36, "mean": 10.027777777777779, "median": 5.0, "p90": 19.5 }, "actions": { "n": 36, "mean": 9.527777777777779, "median": 5.0, "p90": 17.5 }, "valid_actions": { "n": 36, "mean": 7.555555555555555, "median": 5.0, "p90": 13.0 }, "tokens_per_turn": { "n": 36, "mean": 291.91855102552347, "median": 263.325, "p90": 399.6875 }, "generated_tokens": { "n": 36, "mean": 2896.6944444444443, "median": 2517.0, "p90": 5412.5 }, "first_turn_tokens": { "n": 36, "mean": 174.33333333333334, "median": 180.5, "p90": 213.0 }, "searches": { "n": 36, "mean": 2.3055555555555554, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 36, "mean": 2.3055555555555554, "median": 1.0, "p90": 5.0 }, "product_opens": { "n": 36, "mean": 2.111111111111111, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 36, "mean": 1.1944444444444444, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 36, "mean": 0.9166666666666666, "median": 0.0, "p90": 3.0 }, "backtracks": { "n": 36, "mean": 1.3055555555555556, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 36, "mean": 0.05555555555555555, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 36, "mean": 1.1111111111111112, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 36, "mean": 0.3055555555555556, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 36, "mean": 0.16666666666666666, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 36, "mean": 1.9722222222222223, "median": 0.0, "p90": 5.0 }, "reasoning_chars_per_turn": { "n": 36, "mean": 386.60085082484784, "median": 348.11428571428576, "p90": 521.4821428571429 }, "first_product_rank": { "n": 36, "mean": 1.0833333333333333, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.01984126984126984, "pooled_tokens_per_turn": 288.86703601108036, "token_complete_episodes": 36, "rates": { "protocol_errors": 0.1388888888888889, "product_revisits": 0.4444444444444444, "option_changes": 0.08333333333333333, "backtracks": 0.4444444444444444 }, "terminations": { "generation_length": 12, "purchase": 24 } } }, { "step": 67, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.96875, "median": 4.0, "p90": 7.5 }, "actions": { "n": 256, "mean": 4.8359375, "median": 4.0, "p90": 7.0 }, "valid_actions": { "n": 256, "mean": 4.8203125, "median": 4.0, "p90": 7.0 }, "tokens_per_turn": { "n": 256, "mean": 216.45847822718622, "median": 210.625, "p90": 287.125 }, "generated_tokens": { "n": 256, "mean": 1069.23046875, "median": 925.5, "p90": 1521.5 }, "first_turn_tokens": { "n": 256, "mean": 181.91796875, "median": 181.5, "p90": 244.5 }, "searches": { "n": 256, "mean": 1.16015625, "median": 1.0, "p90": 1.5 }, "unique_queries": { "n": 256, "mean": 1.15625, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.10546875, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.05078125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.0546875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.16015625, "median": 0.0, "p90": 0.5 }, "pagination": { "n": 256, "mean": 0.0546875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.359375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.11328125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 283.2969371985151, "median": 273.375, "p90": 385.82894736842104 }, "first_product_rank": { "n": 256, "mean": 1.16015625, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.8984375, "reward_mean": 0.9033203125, "pooled_tokens_per_turn": 215.19103773584905, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.08984375, "product_revisits": 0.04296875, "option_changes": 0.02734375, "backtracks": 0.1015625 }, "terminations": { "purchase": 251, "generation_length": 5 } }, "success": { "n": 230, "tasks": 32, "metrics": { "model_steps": { "n": 230, "mean": 4.7043478260869565, "median": 4.0, "p90": 5.099999999999994 }, "actions": { "n": 230, "mean": 4.626086956521739, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 230, "mean": 4.621739130434783, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 230, "mean": 214.795372803525, "median": 206.95, "p90": 285.42499999999995 }, "generated_tokens": { "n": 230, "mean": 997.7260869565217, "median": 906.5, "p90": 1421.2 }, "first_turn_tokens": { "n": 230, "mean": 181.68695652173912, "median": 181.0, "p90": 248.2 }, "searches": { "n": 230, "mean": 1.108695652173913, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 230, "mean": 1.1043478260869566, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 230, "mean": 1.0739130434782609, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 230, "mean": 1.0478260869565217, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 230, "mean": 0.02608695652173913, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 230, "mean": 0.10869565217391304, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 230, "mean": 0.013043478260869565, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 230, "mean": 1.317391304347826, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 230, "mean": 0.034782608695652174, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 230, "mean": 0.0782608695652174, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 230, "mean": 0.004347826086956522, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 230, "mean": 280.40114654669003, "median": 271.375, "p90": 381.525 }, "first_product_rank": { "n": 230, "mean": 1.0913043478260869, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 212.08595194085026, "token_complete_episodes": 230, "rates": { "protocol_errors": 0.06521739130434782, "product_revisits": 0.021739130434782608, "option_changes": 0.02608695652173913, "backtracks": 0.08260869565217391 }, "terminations": { "purchase": 230 } }, "failure": { "n": 26, "tasks": 8, "metrics": { "model_steps": { "n": 26, "mean": 7.3076923076923075, "median": 5.0, "p90": 15.5 }, "actions": { "n": 26, "mean": 6.6923076923076925, "median": 5.0, "p90": 13.5 }, "valid_actions": { "n": 26, "mean": 6.576923076923077, "median": 5.0, "p90": 13.0 }, "tokens_per_turn": { "n": 26, "mean": 231.17056466726626, "median": 229.92857142857144, "p90": 291.0288461538462 }, "generated_tokens": { "n": 26, "mean": 1701.7692307692307, "median": 1143.5, "p90": 3852.5 }, "first_turn_tokens": { "n": 26, "mean": 183.96153846153845, "median": 186.0, "p90": 242.0 }, "searches": { "n": 26, "mean": 1.6153846153846154, "median": 1.0, "p90": 3.5 }, "unique_queries": { "n": 26, "mean": 1.6153846153846154, "median": 1.0, "p90": 3.5 }, "product_opens": { "n": 26, "mean": 1.3846153846153846, "median": 1.0, "p90": 2.5 }, "unique_products": { "n": 26, "mean": 1.0769230769230769, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 26, "mean": 0.3076923076923077, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 26, "mean": 0.6153846153846154, "median": 0.0, "p90": 2.5 }, "pagination": { "n": 26, "mean": 0.4230769230769231, "median": 0.0, "p90": 2.0 }, "option_selections": { "n": 26, "mean": 1.7307692307692308, "median": 2.0, "p90": 2.5 }, "option_changes": { "n": 26, "mean": 0.038461538461538464, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 26, "mean": 0.4230769230769231, "median": 0.0, "p90": 1.5 }, "invalid_actions": { "n": 26, "mean": 0.11538461538461539, "median": 0.0, "p90": 0.5 }, "reasoning_chars_per_turn": { "n": 26, "mean": 308.913546810814, "median": 300.9, "p90": 406.55 }, "first_product_rank": { "n": 26, "mean": 1.7692307692307692, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.04807692307692308, "pooled_tokens_per_turn": 232.8736842105263, "token_complete_episodes": 26, "rates": { "protocol_errors": 0.3076923076923077, "product_revisits": 0.23076923076923078, "option_changes": 0.038461538461538464, "backtracks": 0.2692307692307692 }, "terminations": { "purchase": 21, "generation_length": 5 } } }, { "step": 68, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.14453125, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 5.0, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.76953125, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 214.6609428207275, "median": 202.125, "p90": 285.625 }, "generated_tokens": { "n": 256, "mean": 1132.0390625, "median": 855.0, "p90": 1353.5 }, "first_turn_tokens": { "n": 256, "mean": 188.05859375, "median": 189.0, "p90": 242.5 }, "searches": { "n": 256, "mean": 1.234375, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.2265625, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.15625, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.05859375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.09765625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.23828125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.08203125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.11328125, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.08984375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.23046875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 273.2235916148943, "median": 257.0, "p90": 376.125 }, "first_product_rank": { "n": 254, "mean": 1.31496062992126, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.81640625, "reward_mean": 0.822265625, "pooled_tokens_per_turn": 220.0470766894457, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.05078125, "product_revisits": 0.046875, "option_changes": 0.0078125, "backtracks": 0.08984375 }, "terminations": { "purchase": 242, "generation_length": 14 } }, "success": { "n": 209, "tasks": 28, "metrics": { "model_steps": { "n": 209, "mean": 4.258373205741627, "median": 4.0, "p90": 5.0 }, "actions": { "n": 209, "mean": 4.22488038277512, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 209, "mean": 4.22488038277512, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 209, "mean": 206.912417072092, "median": 196.75, "p90": 284.5 }, "generated_tokens": { "n": 209, "mean": 876.8803827751196, "median": 813.0, "p90": 1168.0000000000002 }, "first_turn_tokens": { "n": 209, "mean": 186.1244019138756, "median": 187.0, "p90": 235.80000000000007 }, "searches": { "n": 209, "mean": 1.0478468899521531, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 209, "mean": 1.0478468899521531, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 209, "mean": 1.0334928229665072, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 209, "mean": 1.0334928229665072, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 209, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 209, "mean": 0.04784688995215311, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 209, "mean": 0.009569377990430622, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 209, "mean": 1.0861244019138756, "median": 1.0, "p90": 1.0 }, "option_changes": { "n": 209, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 209, "mean": 0.03349282296650718, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 209, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 209, "mean": 267.9766032728881, "median": 249.625, "p90": 375.75 }, "first_product_rank": { "n": 209, "mean": 1.0813397129186604, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 205.9191011235955, "token_complete_episodes": 209, "rates": { "protocol_errors": 0.019138755980861243, "product_revisits": 0.0, "option_changes": 0.0, "backtracks": 0.03349282296650718 }, "terminations": { "purchase": 209 } }, "failure": { "n": 47, "tasks": 12, "metrics": { "model_steps": { "n": 47, "mean": 9.085106382978724, "median": 5.0, "p90": 20.0 }, "actions": { "n": 47, "mean": 8.446808510638299, "median": 5.0, "p90": 19.0 }, "valid_actions": { "n": 47, "mean": 7.191489361702128, "median": 5.0, "p90": 14.0 }, "tokens_per_turn": { "n": 47, "mean": 249.11715306466002, "median": 227.77777777777777, "p90": 306.95789473684204 }, "generated_tokens": { "n": 47, "mean": 2266.68085106383, "median": 1036.0, "p90": 5401.8 }, "first_turn_tokens": { "n": 47, "mean": 196.6595744680851, "median": 203.0, "p90": 249.4 }, "searches": { "n": 47, "mean": 2.0638297872340425, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 47, "mean": 2.021276595744681, "median": 1.0, "p90": 5.0 }, "product_opens": { "n": 47, "mean": 1.702127659574468, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 47, "mean": 1.1702127659574468, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 47, "mean": 0.5319148936170213, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 47, "mean": 1.0851063829787233, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 47, "mean": 0.40425531914893614, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 47, "mean": 1.2340425531914894, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 47, "mean": 0.06382978723404255, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 47, "mean": 0.3404255319148936, "median": 0.0, "p90": 2.0 }, "invalid_actions": { "n": 47, "mean": 1.2553191489361701, "median": 0.0, "p90": 4.399999999999999 }, "reasoning_chars_per_turn": { "n": 47, "mean": 296.5559440293474, "median": 296.7083333333333, "p90": 373.45 }, "first_product_rank": { "n": 45, "mean": 2.4, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.031914893617021274, "pooled_tokens_per_turn": 249.49414519906324, "token_complete_episodes": 47, "rates": { "protocol_errors": 0.19148936170212766, "product_revisits": 0.2553191489361702, "option_changes": 0.0425531914893617, "backtracks": 0.3404255319148936 }, "terminations": { "generation_length": 14, "purchase": 33 } } }, { "step": 69, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.890625, "median": 4.0, "p90": 6.0 }, "actions": { "n": 256, "mean": 4.8203125, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 256, "mean": 4.78515625, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 256, "mean": 236.07097934107355, "median": 201.875, "p90": 295.5 }, "generated_tokens": { "n": 256, "mean": 1152.49609375, "median": 848.0, "p90": 1977.0 }, "first_turn_tokens": { "n": 256, "mean": 181.875, "median": 178.0, "p90": 248.5 }, "searches": { "n": 256, "mean": 1.13671875, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.12109375, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.0859375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0546875, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.140625, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.05859375, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.390625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.1484375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.04296875, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 298.4486218830572, "median": 259.75, "p90": 396.975 }, "first_product_rank": { "n": 255, "mean": 1.0431372549019609, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.87890625, "reward_mean": 0.8914620535714286, "pooled_tokens_per_turn": 235.6541533546326, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.03515625, "product_revisits": 0.02734375, "option_changes": 0.06640625, "backtracks": 0.08203125 }, "terminations": { "purchase": 249, "generation_length": 7 } }, "success": { "n": 225, "tasks": 31, "metrics": { "model_steps": { "n": 225, "mean": 4.502222222222223, "median": 4.0, "p90": 5.0 }, "actions": { "n": 225, "mean": 4.484444444444445, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 225, "mean": 4.4622222222222225, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 225, "mean": 221.37584271284274, "median": 196.0, "p90": 286.15 }, "generated_tokens": { "n": 225, "mean": 1004.72, "median": 825.0, "p90": 1402.7999999999997 }, "first_turn_tokens": { "n": 225, "mean": 183.21333333333334, "median": 180.0, "p90": 248.6 }, "searches": { "n": 225, "mean": 1.0355555555555556, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 225, "mean": 1.0355555555555556, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 225, "mean": 1.031111111111111, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 225, "mean": 1.008888888888889, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 225, "mean": 0.022222222222222223, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 225, "mean": 0.04, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 225, "mean": 0.02666666666666667, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 225, "mean": 1.3288888888888888, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 225, "mean": 0.12, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 225, "mean": 0.017777777777777778, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 225, "mean": 0.022222222222222223, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 225, "mean": 291.9216753246753, "median": 253.25, "p90": 383.24 }, "first_product_rank": { "n": 225, "mean": 1.0355555555555556, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 223.1609081934847, "token_complete_episodes": 225, "rates": { "protocol_errors": 0.017777777777777778, "product_revisits": 0.022222222222222223, "option_changes": 0.057777777777777775, "backtracks": 0.03111111111111111 }, "terminations": { "purchase": 225 } }, "failure": { "n": 31, "tasks": 9, "metrics": { "model_steps": { "n": 31, "mean": 7.709677419354839, "median": 5.0, "p90": 16.0 }, "actions": { "n": 31, "mean": 7.258064516129032, "median": 5.0, "p90": 15.0 }, "valid_actions": { "n": 31, "mean": 7.129032258064516, "median": 5.0, "p90": 15.0 }, "tokens_per_turn": { "n": 31, "mean": 342.72922906210385, "median": 244.5, "p90": 440.2142857142857 }, "generated_tokens": { "n": 31, "mean": 2225.064516129032, "median": 1696.0, "p90": 4588.0 }, "first_turn_tokens": { "n": 31, "mean": 172.16129032258064, "median": 163.0, "p90": 239.0 }, "searches": { "n": 31, "mean": 1.8709677419354838, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 31, "mean": 1.7419354838709677, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 31, "mean": 1.4838709677419355, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 31, "mean": 1.3870967741935485, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 31, "mean": 0.0967741935483871, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 31, "mean": 0.8709677419354839, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 31, "mean": 0.2903225806451613, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 31, "mean": 1.8387096774193548, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 31, "mean": 0.3548387096774194, "median": 0.0, "p90": 1.0 }, "protocol_errors": { "n": 31, "mean": 0.22580645161290322, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 31, "mean": 0.12903225806451613, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 31, "mean": 345.8216210971192, "median": 314.4375, "p90": 469.2 }, "first_product_rank": { "n": 30, "mean": 1.1, "median": 1.0, "p90": 1.1000000000000014 } }, "success_rate": 0.0, "reward_mean": 0.10368663594470047, "pooled_tokens_per_turn": 288.60669456066944, "token_complete_episodes": 31, "rates": { "protocol_errors": 0.16129032258064516, "product_revisits": 0.06451612903225806, "option_changes": 0.12903225806451613, "backtracks": 0.45161290322580644 }, "terminations": { "purchase": 24, "generation_length": 7 } } }, { "step": 70, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.41796875, "median": 4.0, "p90": 9.0 }, "actions": { "n": 256, "mean": 5.2421875, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 256, "mean": 5.22265625, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 256, "mean": 231.75170057297686, "median": 210.25, "p90": 304.5 }, "generated_tokens": { "n": 256, "mean": 1315.24609375, "median": 874.5, "p90": 2733.0 }, "first_turn_tokens": { "n": 256, "mean": 179.78125, "median": 177.5, "p90": 230.0 }, "searches": { "n": 256, "mean": 1.33203125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.3203125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.24609375, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.12109375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.33984375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.06640625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.28515625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.109375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.12890625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 302.4567671831717, "median": 275.0, "p90": 437.5 }, "first_product_rank": { "n": 255, "mean": 1.4941176470588236, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.79296875, "reward_mean": 0.79296875, "pooled_tokens_per_turn": 242.75630857966834, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.08203125, "product_revisits": 0.09765625, "option_changes": 0.046875, "backtracks": 0.140625 }, "terminations": { "purchase": 244, "generation_length": 12 } }, "success": { "n": 203, "tasks": 27, "metrics": { "model_steps": { "n": 203, "mean": 4.610837438423645, "median": 4.0, "p90": 5.0 }, "actions": { "n": 203, "mean": 4.5763546798029555, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 203, "mean": 4.571428571428571, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 203, "mean": 217.3718440563268, "median": 206.0, "p90": 291.6 }, "generated_tokens": { "n": 203, "mean": 1029.9950738916257, "median": 835.0, "p90": 1485.2000000000005 }, "first_turn_tokens": { "n": 203, "mean": 181.11330049261085, "median": 179.0, "p90": 228.0 }, "searches": { "n": 203, "mean": 1.1330049261083743, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 203, "mean": 1.1280788177339902, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 203, "mean": 1.1133004926108374, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 203, "mean": 1.0492610837438423, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 203, "mean": 0.06403940886699508, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 203, "mean": 0.1330049261083744, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 203, "mean": 0.024630541871921183, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 203, "mean": 1.167487684729064, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 203, "mean": 0.04926108374384237, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 203, "mean": 0.034482758620689655, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 203, "mean": 0.0049261083743842365, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 203, "mean": 289.11275705684574, "median": 265.75, "p90": 402.6333333333334 }, "first_product_rank": { "n": 203, "mean": 1.2660098522167487, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 223.38568376068375, "token_complete_episodes": 203, "rates": { "protocol_errors": 0.029556650246305417, "product_revisits": 0.054187192118226604, "option_changes": 0.029556650246305417, "backtracks": 0.07389162561576355 }, "terminations": { "purchase": 203 } }, "failure": { "n": 53, "tasks": 9, "metrics": { "model_steps": { "n": 53, "mean": 8.50943396226415, "median": 5.0, "p90": 18.0 }, "actions": { "n": 53, "mean": 7.7924528301886795, "median": 5.0, "p90": 16.0 }, "valid_actions": { "n": 53, "mean": 7.716981132075472, "median": 5.0, "p90": 16.0 }, "tokens_per_turn": { "n": 53, "mean": 286.8292642122214, "median": 252.17391304347825, "p90": 332.95000000000005 }, "generated_tokens": { "n": 53, "mean": 2407.811320754717, "median": 1306.0, "p90": 5347.400000000001 }, "first_turn_tokens": { "n": 53, "mean": 174.67924528301887, "median": 171.0, "p90": 245.8 }, "searches": { "n": 53, "mean": 2.0943396226415096, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 53, "mean": 2.056603773584906, "median": 1.0, "p90": 5.0 }, "product_opens": { "n": 53, "mean": 1.7547169811320755, "median": 1.0, "p90": 3.8000000000000043 }, "unique_products": { "n": 53, "mean": 1.3962264150943395, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 53, "mean": 0.3584905660377358, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 53, "mean": 1.1320754716981132, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 53, "mean": 0.22641509433962265, "median": 0.0, "p90": 0.8000000000000043 }, "option_selections": { "n": 53, "mean": 1.7358490566037736, "median": 1.0, "p90": 4.0 }, "option_changes": { "n": 53, "mean": 0.33962264150943394, "median": 0.0, "p90": 0.8000000000000043 }, "protocol_errors": { "n": 53, "mean": 0.49056603773584906, "median": 0.0, "p90": 1.8000000000000043 }, "invalid_actions": { "n": 53, "mean": 0.07547169811320754, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 53, "mean": 353.5668437047599, "median": 363.88235294117646, "p90": 506.53333333333336 }, "first_product_rank": { "n": 52, "mean": 2.3846153846153846, "median": 1.0, "p90": 5.899999999999999 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 282.95787139689577, "token_complete_episodes": 53, "rates": { "protocol_errors": 0.2830188679245283, "product_revisits": 0.2641509433962264, "option_changes": 0.11320754716981132, "backtracks": 0.39622641509433965 }, "terminations": { "purchase": 41, "generation_length": 12 } } }, { "step": 71, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.47265625, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.38671875, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.3359375, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 214.00502596872082, "median": 209.375, "p90": 278.5 }, "generated_tokens": { "n": 256, "mean": 978.234375, "median": 858.0, "p90": 1269.0 }, "first_turn_tokens": { "n": 256, "mean": 172.5625, "median": 160.0, "p90": 237.0 }, "searches": { "n": 256, "mean": 1.07421875, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.07421875, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.0546875, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.015625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.0390625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.12890625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0703125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 282.8750196041958, "median": 273.125, "p90": 374.025 }, "first_product_rank": { "n": 256, "mean": 1.125, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.9296875, "reward_mean": 0.9296875, "pooled_tokens_per_turn": 218.71441048034936, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0703125, "product_revisits": 0.0234375, "option_changes": 0.015625, "backtracks": 0.0390625 }, "terminations": { "purchase": 252, "generation_length": 4 } }, "success": { "n": 238, "tasks": 31, "metrics": { "model_steps": { "n": 238, "mean": 4.2521008403361344, "median": 4.0, "p90": 5.0 }, "actions": { "n": 238, "mean": 4.189075630252101, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 238, "mean": 4.189075630252101, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 238, "mean": 213.94652360944377, "median": 209.375, "p90": 274.07 }, "generated_tokens": { "n": 238, "mean": 916.1008403361344, "median": 857.0, "p90": 1239.3000000000002 }, "first_turn_tokens": { "n": 238, "mean": 173.3109243697479, "median": 162.0, "p90": 237.0 }, "searches": { "n": 238, "mean": 1.0168067226890756, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 238, "mean": 1.0168067226890756, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 238, "mean": 1.0168067226890756, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 238, "mean": 1.0084033613445378, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 238, "mean": 0.008403361344537815, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 238, "mean": 0.01680672268907563, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 238, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 238, "mean": 1.138655462184874, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 238, "mean": 0.025210084033613446, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 238, "mean": 0.06302521008403361, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 238, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 238, "mean": 282.5903378017874, "median": 273.125, "p90": 373.24 }, "first_product_rank": { "n": 238, "mean": 1.1218487394957983, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 215.44664031620553, "token_complete_episodes": 238, "rates": { "protocol_errors": 0.06302521008403361, "product_revisits": 0.004201680672268907, "option_changes": 0.01680672268907563, "backtracks": 0.012605042016806723 }, "terminations": { "purchase": 238 } }, "failure": { "n": 18, "tasks": 6, "metrics": { "model_steps": { "n": 18, "mean": 7.388888888888889, "median": 4.0, "p90": 15.600000000000001 }, "actions": { "n": 18, "mean": 7.0, "median": 4.0, "p90": 14.600000000000001 }, "valid_actions": { "n": 18, "mean": 6.277777777777778, "median": 4.0, "p90": 11.3 }, "tokens_per_turn": { "n": 18, "mean": 214.77855716360617, "median": 207.5, "p90": 290.49926470588235 }, "generated_tokens": { "n": 18, "mean": 1799.7777777777778, "median": 883.5, "p90": 4401.500000000001 }, "first_turn_tokens": { "n": 18, "mean": 162.66666666666666, "median": 152.5, "p90": 221.20000000000002 }, "searches": { "n": 18, "mean": 1.8333333333333333, "median": 1.0, "p90": 4.0 }, "unique_queries": { "n": 18, "mean": 1.8333333333333333, "median": 1.0, "p90": 4.0 }, "product_opens": { "n": 18, "mean": 1.5555555555555556, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 18, "mean": 1.1111111111111112, "median": 1.0, "p90": 1.3000000000000007 }, "product_revisits": { "n": 18, "mean": 0.4444444444444444, "median": 0.0, "p90": 1.3000000000000007 }, "backtracks": { "n": 18, "mean": 0.8888888888888888, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 18, "mean": 0.2222222222222222, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 18, "mean": 1.0, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 18, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 18, "mean": 0.16666666666666666, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 18, "mean": 0.7222222222222222, "median": 0.0, "p90": 3.3000000000000007 }, "reasoning_chars_per_turn": { "n": 18, "mean": 286.6391456582634, "median": 278.25, "p90": 386.2166666666667 }, "first_product_rank": { "n": 18, "mean": 1.1666666666666667, "median": 1.0, "p90": 1.3000000000000007 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 243.57894736842104, "token_complete_episodes": 18, "rates": { "protocol_errors": 0.16666666666666666, "product_revisits": 0.2777777777777778, "option_changes": 0.0, "backtracks": 0.3888888888888889 }, "terminations": { "purchase": 14, "generation_length": 4 } } }, { "step": 72, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 6.45703125, "median": 4.0, "p90": 13.5 }, "actions": { "n": 256, "mean": 6.125, "median": 4.0, "p90": 12.5 }, "valid_actions": { "n": 256, "mean": 5.9453125, "median": 4.0, "p90": 11.0 }, "tokens_per_turn": { "n": 256, "mean": 255.74096453670938, "median": 231.4375, "p90": 351.7232142857143 }, "generated_tokens": { "n": 256, "mean": 1707.515625, "median": 1083.0, "p90": 4330.0 }, "first_turn_tokens": { "n": 256, "mean": 174.109375, "median": 167.0, "p90": 239.0 }, "searches": { "n": 256, "mean": 1.51953125, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 256, "mean": 1.51171875, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 256, "mean": 1.42578125, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 256, "mean": 1.2421875, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 256, "mean": 0.18359375, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 256, "mean": 0.51953125, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 256, "mean": 0.02734375, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.54296875, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 256, "mean": 0.07421875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.2421875, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.1796875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 339.0776566180774, "median": 305.775, "p90": 473.19117647058823 }, "first_product_rank": { "n": 256, "mean": 1.3203125, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.77734375, "reward_mean": 0.78759765625, "pooled_tokens_per_turn": 264.44283121597095, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.16796875, "product_revisits": 0.1171875, "option_changes": 0.0625, "backtracks": 0.2265625 }, "terminations": { "generation_length": 23, "purchase": 233 } }, "success": { "n": 199, "tasks": 30, "metrics": { "model_steps": { "n": 199, "mean": 4.939698492462312, "median": 4.0, "p90": 6.200000000000017 }, "actions": { "n": 199, "mean": 4.839195979899498, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 199, "mean": 4.839195979899498, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 199, "mean": 228.82743318936787, "median": 219.5, "p90": 302.3 }, "generated_tokens": { "n": 199, "mean": 1155.0804020100502, "median": 992.0, "p90": 1740.4000000000003 }, "first_turn_tokens": { "n": 199, "mean": 175.5075376884422, "median": 167.0, "p90": 244.0 }, "searches": { "n": 199, "mean": 1.1306532663316582, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 199, "mean": 1.1306532663316582, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 199, "mean": 1.1306532663316582, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 199, "mean": 1.100502512562814, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 199, "mean": 0.03015075376884422, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 199, "mean": 0.1306532663316583, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 199, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 199, "mean": 1.4472361809045227, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 199, "mean": 0.05527638190954774, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 199, "mean": 0.10050251256281408, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 199, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 199, "mean": 306.9339984973151, "median": 292.75, "p90": 407.6 }, "first_product_rank": { "n": 199, "mean": 1.065326633165829, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 233.83621566632758, "token_complete_episodes": 199, "rates": { "protocol_errors": 0.09547738693467336, "product_revisits": 0.02512562814070352, "option_changes": 0.05527638190954774, "backtracks": 0.09547738693467336 }, "terminations": { "purchase": 199 } }, "failure": { "n": 57, "tasks": 11, "metrics": { "model_steps": { "n": 57, "mean": 11.75438596491228, "median": 12.0, "p90": 19.0 }, "actions": { "n": 57, "mean": 10.614035087719298, "median": 11.0, "p90": 17.0 }, "valid_actions": { "n": 57, "mean": 9.807017543859649, "median": 10.0, "p90": 16.0 }, "tokens_per_turn": { "n": 57, "mean": 349.70224064409473, "median": 300.5, "p90": 589.8999999999999 }, "generated_tokens": { "n": 57, "mean": 3636.1929824561403, "median": 3925.0, "p90": 5506.0 }, "first_turn_tokens": { "n": 57, "mean": 169.2280701754386, "median": 172.0, "p90": 210.4 }, "searches": { "n": 57, "mean": 2.8771929824561404, "median": 3.0, "p90": 5.399999999999999 }, "unique_queries": { "n": 57, "mean": 2.8421052631578947, "median": 3.0, "p90": 5.0 }, "product_opens": { "n": 57, "mean": 2.456140350877193, "median": 2.0, "p90": 4.0 }, "unique_products": { "n": 57, "mean": 1.736842105263158, "median": 2.0, "p90": 3.0 }, "product_revisits": { "n": 57, "mean": 0.7192982456140351, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 57, "mean": 1.8771929824561404, "median": 2.0, "p90": 4.399999999999999 }, "pagination": { "n": 57, "mean": 0.12280701754385964, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 57, "mean": 1.8771929824561404, "median": 2.0, "p90": 3.3999999999999986 }, "option_changes": { "n": 57, "mean": 0.14035087719298245, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 57, "mean": 0.7368421052631579, "median": 0.0, "p90": 2.0 }, "invalid_actions": { "n": 57, "mean": 0.8070175438596491, "median": 0.0, "p90": 3.0 }, "reasoning_chars_per_turn": { "n": 57, "mean": 451.29849812740554, "median": 395.63157894736844, "p90": 696.2424242424241 }, "first_product_rank": { "n": 57, "mean": 2.210526315789474, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.0, "reward_mean": 0.046052631578947366, "pooled_tokens_per_turn": 309.34776119402983, "token_complete_episodes": 57, "rates": { "protocol_errors": 0.42105263157894735, "product_revisits": 0.43859649122807015, "option_changes": 0.08771929824561403, "backtracks": 0.6842105263157895 }, "terminations": { "generation_length": 23, "purchase": 34 } } }, { "step": 73, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.234375, "median": 4.0, "p90": 7.5 }, "actions": { "n": 256, "mean": 5.0234375, "median": 4.0, "p90": 6.5 }, "valid_actions": { "n": 256, "mean": 4.9921875, "median": 4.0, "p90": 6.5 }, "tokens_per_turn": { "n": 256, "mean": 228.39865266186783, "median": 205.375, "p90": 317.1 }, "generated_tokens": { "n": 256, "mean": 1280.33203125, "median": 895.0, "p90": 2250.0 }, "first_turn_tokens": { "n": 256, "mean": 169.34375, "median": 164.0, "p90": 216.0 }, "searches": { "n": 256, "mean": 1.17578125, "median": 1.0, "p90": 1.5 }, "unique_queries": { "n": 256, "mean": 1.171875, "median": 1.0, "p90": 1.5 }, "product_opens": { "n": 256, "mean": 1.1484375, "median": 1.0, "p90": 1.5 }, "unique_products": { "n": 256, "mean": 1.06640625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.08203125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.17578125, "median": 0.0, "p90": 0.5 }, "pagination": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.49609375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.03515625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.17578125, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 298.38593571684396, "median": 270.2, "p90": 436.125 }, "first_product_rank": { "n": 256, "mean": 1.328125, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.89453125, "reward_mean": 0.89453125, "pooled_tokens_per_turn": 244.6007462686567, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.1484375, "product_revisits": 0.0625, "option_changes": 0.01953125, "backtracks": 0.1015625 }, "terminations": { "purchase": 247, "generation_length": 9 } }, "success": { "n": 229, "tasks": 29, "metrics": { "model_steps": { "n": 229, "mean": 4.598253275109171, "median": 4.0, "p90": 5.0 }, "actions": { "n": 229, "mean": 4.502183406113537, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 229, "mean": 4.489082969432315, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 229, "mean": 209.19843522561862, "median": 197.5, "p90": 275.40000000000003 }, "generated_tokens": { "n": 229, "mean": 968.1222707423581, "median": 862.0, "p90": 1252.8000000000002 }, "first_turn_tokens": { "n": 229, "mean": 169.4061135371179, "median": 164.0, "p90": 216.40000000000003 }, "searches": { "n": 229, "mean": 1.0436681222707425, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 229, "mean": 1.0436681222707425, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 229, "mean": 1.0436681222707425, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 229, "mean": 1.039301310043668, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 229, "mean": 0.004366812227074236, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 229, "mean": 0.043668122270742356, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 229, "mean": 0.004366812227074236, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 229, "mean": 1.3537117903930131, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 229, "mean": 0.008733624454148471, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 229, "mean": 0.09606986899563319, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 229, "mean": 0.013100436681222707, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 229, "mean": 275.21369480834545, "median": 260.25, "p90": 374.2266666666667 }, "first_product_rank": { "n": 229, "mean": 1.331877729257642, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 210.54131054131054, "token_complete_episodes": 229, "rates": { "protocol_errors": 0.09170305676855896, "product_revisits": 0.004366812227074236, "option_changes": 0.004366812227074236, "backtracks": 0.039301310043668124 }, "terminations": { "purchase": 229 } }, "failure": { "n": 27, "tasks": 6, "metrics": { "model_steps": { "n": 27, "mean": 10.62962962962963, "median": 12.0, "p90": 17.0 }, "actions": { "n": 27, "mean": 9.444444444444445, "median": 11.0, "p90": 14.0 }, "valid_actions": { "n": 27, "mean": 9.25925925925926, "median": 10.0, "p90": 13.400000000000002 }, "tokens_per_turn": { "n": 27, "mean": 391.2449412878334, "median": 335.64285714285717, "p90": 579.1807692307692 }, "generated_tokens": { "n": 27, "mean": 3928.3333333333335, "median": 3825.0, "p90": 7241.400000000001 }, "first_turn_tokens": { "n": 27, "mean": 168.8148148148148, "median": 164.0, "p90": 207.4 }, "searches": { "n": 27, "mean": 2.2962962962962963, "median": 2.0, "p90": 4.0 }, "unique_queries": { "n": 27, "mean": 2.259259259259259, "median": 2.0, "p90": 4.0 }, "product_opens": { "n": 27, "mean": 2.037037037037037, "median": 2.0, "p90": 3.400000000000002 }, "unique_products": { "n": 27, "mean": 1.2962962962962963, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 27, "mean": 0.7407407407407407, "median": 1.0, "p90": 2.0 }, "backtracks": { "n": 27, "mean": 1.2962962962962963, "median": 1.0, "p90": 3.0 }, "pagination": { "n": 27, "mean": 0.25925925925925924, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 27, "mean": 2.7037037037037037, "median": 2.0, "p90": 4.400000000000002 }, "option_changes": { "n": 27, "mean": 0.25925925925925924, "median": 0.0, "p90": 1.4000000000000021 }, "protocol_errors": { "n": 27, "mean": 0.8518518518518519, "median": 1.0, "p90": 2.0 }, "invalid_actions": { "n": 27, "mean": 0.18518518518518517, "median": 0.0, "p90": 0.40000000000000213 }, "reasoning_chars_per_turn": { "n": 27, "mean": 494.92086786670114, "median": 456.625, "p90": 673.1164835164835 }, "first_product_rank": { "n": 27, "mean": 1.2962962962962963, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 369.5644599303136, "token_complete_episodes": 27, "rates": { "protocol_errors": 0.6296296296296297, "product_revisits": 0.5555555555555556, "option_changes": 0.14814814814814814, "backtracks": 0.6296296296296297 }, "terminations": { "purchase": 18, "generation_length": 9 } } }, { "step": 74, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.5390625, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.39453125, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.390625, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 203.91581386278654, "median": 194.675, "p90": 274.83333333333337 }, "generated_tokens": { "n": 256, "mean": 940.71484375, "median": 824.0, "p90": 1273.0 }, "first_turn_tokens": { "n": 256, "mean": 169.48046875, "median": 164.0, "p90": 226.0 }, "searches": { "n": 256, "mean": 1.03125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.03125, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.0234375, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0078125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.01953125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.2890625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.05078125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.140625, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.00390625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 267.1449002494901, "median": 253.98333333333335, "p90": 369.125 }, "first_product_rank": { "n": 255, "mean": 1.0392156862745099, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.96484375, "reward_mean": 0.96484375, "pooled_tokens_per_turn": 207.2487091222031, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.13671875, "product_revisits": 0.015625, "option_changes": 0.03125, "backtracks": 0.02734375 }, "terminations": { "purchase": 255, "generation_length": 1 } }, "success": { "n": 247, "tasks": 32, "metrics": { "model_steps": { "n": 247, "mean": 4.445344129554656, "median": 4.0, "p90": 5.0 }, "actions": { "n": 247, "mean": 4.3117408906882595, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 247, "mean": 4.3117408906882595, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 247, "mean": 199.8307375373165, "median": 193.25, "p90": 265.35 }, "generated_tokens": { "n": 247, "mean": 893.834008097166, "median": 815.0, "p90": 1182.6000000000001 }, "first_turn_tokens": { "n": 247, "mean": 169.89473684210526, "median": 165.0, "p90": 226.0 }, "searches": { "n": 247, "mean": 1.0202429149797572, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 247, "mean": 1.0202429149797572, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 247, "mean": 1.0202429149797572, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 247, "mean": 1.008097165991903, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 247, "mean": 0.012145748987854251, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 247, "mean": 0.020242914979757085, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 247, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 247, "mean": 1.2510121457489878, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 247, "mean": 0.008097165991902834, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 247, "mean": 0.13360323886639677, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 247, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 247, "mean": 260.6555852042694, "median": 251.5, "p90": 366.5 }, "first_product_rank": { "n": 247, "mean": 1.040485829959514, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 201.0719489981785, "token_complete_episodes": 247, "rates": { "protocol_errors": 0.12955465587044535, "product_revisits": 0.012145748987854251, "option_changes": 0.008097165991902834, "backtracks": 0.020242914979757085 }, "terminations": { "purchase": 247 } }, "failure": { "n": 9, "tasks": 3, "metrics": { "model_steps": { "n": 9, "mean": 7.111111111111111, "median": 6.0, "p90": 11.4 }, "actions": { "n": 9, "mean": 6.666666666666667, "median": 6.0, "p90": 9.600000000000001 }, "valid_actions": { "n": 9, "mean": 6.555555555555555, "median": 6.0, "p90": 8.8 }, "tokens_per_turn": { "n": 9, "mean": 316.02846412846407, "median": 305.1818181818182, "p90": 392.35999999999996 }, "generated_tokens": { "n": 9, "mean": 2227.3333333333335, "median": 1942.0, "p90": 3547.0 }, "first_turn_tokens": { "n": 9, "mean": 158.11111111111111, "median": 156.0, "p90": 181.8 }, "searches": { "n": 9, "mean": 1.3333333333333333, "median": 1.0, "p90": 2.2 }, "unique_queries": { "n": 9, "mean": 1.3333333333333333, "median": 1.0, "p90": 2.2 }, "product_opens": { "n": 9, "mean": 1.1111111111111112, "median": 1.0, "p90": 1.4000000000000004 }, "unique_products": { "n": 9, "mean": 1.0, "median": 1.0, "p90": 1.2000000000000002 }, "product_revisits": { "n": 9, "mean": 0.1111111111111111, "median": 0.0, "p90": 0.20000000000000018 }, "backtracks": { "n": 9, "mean": 0.3333333333333333, "median": 0.0, "p90": 1.2000000000000002 }, "pagination": { "n": 9, "mean": 0.5555555555555556, "median": 0.0, "p90": 1.0000000000000009 }, "option_selections": { "n": 9, "mean": 2.3333333333333335, "median": 2.0, "p90": 3.4000000000000004 }, "option_changes": { "n": 9, "mean": 1.2222222222222223, "median": 1.0, "p90": 2.4000000000000004 }, "protocol_errors": { "n": 9, "mean": 0.3333333333333333, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 9, "mean": 0.1111111111111111, "median": 0.0, "p90": 0.20000000000000018 }, "reasoning_chars_per_turn": { "n": 9, "mean": 445.24054649054654, "median": 447.3636363636364, "p90": 570.36 }, "first_product_rank": { "n": 8, "mean": 1.0, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 313.21875, "token_complete_episodes": 9, "rates": { "protocol_errors": 0.3333333333333333, "product_revisits": 0.1111111111111111, "option_changes": 0.6666666666666666, "backtracks": 0.2222222222222222 }, "terminations": { "purchase": 8, "generation_length": 1 } } }, { "step": 75, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.72265625, "median": 4.0, "p90": 9.5 }, "actions": { "n": 256, "mean": 5.44140625, "median": 4.0, "p90": 9.0 }, "valid_actions": { "n": 256, "mean": 5.40234375, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 256, "mean": 210.25589132403928, "median": 190.775, "p90": 280.5 }, "generated_tokens": { "n": 256, "mean": 1271.95703125, "median": 828.0, "p90": 2780.0 }, "first_turn_tokens": { "n": 256, "mean": 149.37109375, "median": 149.0, "p90": 197.0 }, "searches": { "n": 256, "mean": 1.32421875, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.31640625, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.28125, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.15234375, "median": 1.0, "p90": 1.5 }, "product_revisits": { "n": 256, "mean": 0.12890625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.32421875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.01171875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.49609375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.16015625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.24609375, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.0390625, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 271.87489218861117, "median": 248.5, "p90": 390.375 }, "first_product_rank": { "n": 256, "mean": 1.0625, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.90625, "reward_mean": 0.91015625, "pooled_tokens_per_turn": 222.2668941979522, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.16015625, "product_revisits": 0.06640625, "option_changes": 0.0546875, "backtracks": 0.12890625 }, "terminations": { "purchase": 247, "generation_length": 9 } }, "success": { "n": 232, "tasks": 32, "metrics": { "model_steps": { "n": 232, "mean": 4.698275862068965, "median": 4.0, "p90": 5.0 }, "actions": { "n": 232, "mean": 4.560344827586207, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 232, "mean": 4.547413793103448, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 232, "mean": 199.99947541119576, "median": 187.225, "p90": 262.45 }, "generated_tokens": { "n": 232, "mean": 961.4181034482758, "median": 792.5, "p90": 1378.2000000000003 }, "first_turn_tokens": { "n": 232, "mean": 148.81896551724137, "median": 148.0, "p90": 194.9 }, "searches": { "n": 232, "mean": 1.0948275862068966, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 232, "mean": 1.0948275862068966, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 232, "mean": 1.0905172413793103, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 232, "mean": 1.0732758620689655, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 232, "mean": 0.017241379310344827, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 232, "mean": 0.09482758620689655, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 232, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 232, "mean": 1.2672413793103448, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 232, "mean": 0.08189655172413793, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 232, "mean": 0.13793103448275862, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 232, "mean": 0.01293103448275862, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 232, "mean": 262.8492748813616, "median": 240.75, "p90": 362.87000000000006 }, "first_product_rank": { "n": 232, "mean": 1.0517241379310345, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 204.63211009174313, "token_complete_episodes": 232, "rates": { "protocol_errors": 0.11206896551724138, "product_revisits": 0.01293103448275862, "option_changes": 0.03017241379310345, "backtracks": 0.0603448275862069 }, "terminations": { "purchase": 232 } }, "failure": { "n": 24, "tasks": 8, "metrics": { "model_steps": { "n": 24, "mean": 15.625, "median": 17.0, "p90": 22.7 }, "actions": { "n": 24, "mean": 13.958333333333334, "median": 15.0, "p90": 19.7 }, "valid_actions": { "n": 24, "mean": 13.666666666666666, "median": 15.0, "p90": 19.0 }, "tokens_per_turn": { "n": 24, "mean": 309.40124514819337, "median": 256.98333333333335, "p90": 374.4867647058823 }, "generated_tokens": { "n": 24, "mean": 4273.833333333333, "median": 4616.5, "p90": 6165.9 }, "first_turn_tokens": { "n": 24, "mean": 154.70833333333334, "median": 155.5, "p90": 198.1 }, "searches": { "n": 24, "mean": 3.5416666666666665, "median": 4.0, "p90": 5.0 }, "unique_queries": { "n": 24, "mean": 3.4583333333333335, "median": 4.0, "p90": 5.0 }, "product_opens": { "n": 24, "mean": 3.125, "median": 3.0, "p90": 5.0 }, "unique_products": { "n": 24, "mean": 1.9166666666666667, "median": 2.0, "p90": 3.0 }, "product_revisits": { "n": 24, "mean": 1.2083333333333333, "median": 1.0, "p90": 3.0 }, "backtracks": { "n": 24, "mean": 2.5416666666666665, "median": 3.0, "p90": 4.0 }, "pagination": { "n": 24, "mean": 0.125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 24, "mean": 3.7083333333333335, "median": 3.0, "p90": 7.699999999999999 }, "option_changes": { "n": 24, "mean": 0.9166666666666666, "median": 0.0, "p90": 3.6999999999999993 }, "protocol_errors": { "n": 24, "mean": 1.2916666666666667, "median": 1.0, "p90": 2.0 }, "invalid_actions": { "n": 24, "mean": 0.2916666666666667, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 24, "mean": 359.12252615869056, "median": 361.3875, "p90": 462.80802139037434 }, "first_product_rank": { "n": 24, "mean": 1.1666666666666667, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.0, "reward_mean": 0.041666666666666664, "pooled_tokens_per_turn": 273.5253333333333, "token_complete_episodes": 24, "rates": { "protocol_errors": 0.625, "product_revisits": 0.5833333333333334, "option_changes": 0.2916666666666667, "backtracks": 0.7916666666666666 }, "terminations": { "purchase": 15, "generation_length": 9 } } }, { "step": 76, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.65625, "median": 5.0, "p90": 9.5 }, "actions": { "n": 256, "mean": 5.40625, "median": 4.0, "p90": 9.0 }, "valid_actions": { "n": 256, "mean": 5.34765625, "median": 4.0, "p90": 8.5 }, "tokens_per_turn": { "n": 256, "mean": 227.45400447145892, "median": 189.25, "p90": 303.79230769230765 }, "generated_tokens": { "n": 256, "mean": 1341.078125, "median": 854.5, "p90": 2773.5 }, "first_turn_tokens": { "n": 256, "mean": 155.60546875, "median": 151.0, "p90": 210.0 }, "searches": { "n": 256, "mean": 1.30078125, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.30078125, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.2421875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 256, "mean": 1.13671875, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.10546875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.30859375, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.5234375, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.08203125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.19921875, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.05859375, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 279.552017607027, "median": 243.975, "p90": 412.05128205128204 }, "first_product_rank": { "n": 254, "mean": 1.2283464566929134, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.85546875, "reward_mean": 0.8671875, "pooled_tokens_per_turn": 237.09668508287294, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.140625, "product_revisits": 0.078125, "option_changes": 0.0546875, "backtracks": 0.15234375 }, "terminations": { "purchase": 243, "generation_length": 13 } }, "success": { "n": 219, "tasks": 30, "metrics": { "model_steps": { "n": 219, "mean": 5.073059360730594, "median": 4.0, "p90": 7.0 }, "actions": { "n": 219, "mean": 4.940639269406392, "median": 4.0, "p90": 7.0 }, "valid_actions": { "n": 219, "mean": 4.931506849315069, "median": 4.0, "p90": 7.0 }, "tokens_per_turn": { "n": 219, "mean": 201.40203033752587, "median": 179.0, "p90": 262.76 }, "generated_tokens": { "n": 219, "mean": 1054.867579908676, "median": 794.0, "p90": 1846.0000000000005 }, "first_turn_tokens": { "n": 219, "mean": 155.28310502283105, "median": 151.0, "p90": 210.0 }, "searches": { "n": 219, "mean": 1.1552511415525115, "median": 1.0, "p90": 1.200000000000017 }, "unique_queries": { "n": 219, "mean": 1.1552511415525115, "median": 1.0, "p90": 1.200000000000017 }, "product_opens": { "n": 219, "mean": 1.1506849315068493, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 219, "mean": 1.1050228310502284, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 219, "mean": 0.045662100456621, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 219, "mean": 0.1598173515981735, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 219, "mean": 0.0045662100456621, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 219, "mean": 1.461187214611872, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 219, "mean": 0.0273972602739726, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 219, "mean": 0.1324200913242009, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 219, "mean": 0.0091324200913242, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 219, "mean": 264.69840435551555, "median": 229.0, "p90": 364.8823529411767 }, "first_product_rank": { "n": 219, "mean": 1.1369863013698631, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 207.93519351935194, "token_complete_episodes": 219, "rates": { "protocol_errors": 0.1050228310502283, "product_revisits": 0.0365296803652968, "option_changes": 0.0182648401826484, "backtracks": 0.1050228310502283 }, "terminations": { "purchase": 219 } }, "failure": { "n": 37, "tasks": 11, "metrics": { "model_steps": { "n": 37, "mean": 9.108108108108109, "median": 6.0, "p90": 18.4 }, "actions": { "n": 37, "mean": 8.162162162162161, "median": 6.0, "p90": 16.0 }, "valid_actions": { "n": 37, "mean": 7.8108108108108105, "median": 6.0, "p90": 14.399999999999999 }, "tokens_per_turn": { "n": 37, "mean": 381.6535270479812, "median": 253.94444444444446, "p90": 652.2223776223775 }, "generated_tokens": { "n": 37, "mean": 3035.135135135135, "median": 2065.0, "p90": 6565.2 }, "first_turn_tokens": { "n": 37, "mean": 157.51351351351352, "median": 155.0, "p90": 221.2 }, "searches": { "n": 37, "mean": 2.1621621621621623, "median": 1.0, "p90": 5.0 }, "unique_queries": { "n": 37, "mean": 2.1621621621621623, "median": 1.0, "p90": 5.0 }, "product_opens": { "n": 37, "mean": 1.7837837837837838, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 37, "mean": 1.3243243243243243, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 37, "mean": 0.4594594594594595, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 37, "mean": 1.1891891891891893, "median": 0.0, "p90": 4.0 }, "pagination": { "n": 37, "mean": 0.13513513513513514, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 37, "mean": 1.8918918918918919, "median": 2.0, "p90": 3.0 }, "option_changes": { "n": 37, "mean": 0.40540540540540543, "median": 0.0, "p90": 1.3999999999999986 }, "protocol_errors": { "n": 37, "mean": 0.5945945945945946, "median": 0.0, "p90": 2.0 }, "invalid_actions": { "n": 37, "mean": 0.35135135135135137, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 37, "mean": 367.46935009570285, "median": 331.1111111111111, "p90": 532.197894736842 }, "first_product_rank": { "n": 35, "mean": 1.8, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.0, "reward_mean": 0.08108108108108109, "pooled_tokens_per_turn": 333.23442136498517, "token_complete_episodes": 37, "rates": { "protocol_errors": 0.35135135135135137, "product_revisits": 0.32432432432432434, "option_changes": 0.2702702702702703, "backtracks": 0.43243243243243246 }, "terminations": { "purchase": 24, "generation_length": 13 } } }, { "step": 77, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.1796875, "median": 4.0, "p90": 7.0 }, "actions": { "n": 256, "mean": 4.90625, "median": 4.0, "p90": 6.5 }, "valid_actions": { "n": 256, "mean": 4.8828125, "median": 4.0, "p90": 6.5 }, "tokens_per_turn": { "n": 256, "mean": 228.2747773011779, "median": 208.625, "p90": 315.1875 }, "generated_tokens": { "n": 256, "mean": 1244.0546875, "median": 904.0, "p90": 1744.5 }, "first_turn_tokens": { "n": 256, "mean": 158.3203125, "median": 158.0, "p90": 204.5 }, "searches": { "n": 256, "mean": 1.22265625, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 256, "mean": 1.21875, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 256, "mean": 1.18359375, "median": 1.0, "p90": 1.5 }, "unique_products": { "n": 256, "mean": 1.08203125, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.1015625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.23046875, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.25, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0546875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.23828125, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 256, "mean": 0.0234375, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 296.1460259869311, "median": 272.5, "p90": 418.16666666666663 }, "first_product_rank": { "n": 256, "mean": 1.39453125, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.89453125, "reward_mean": 0.89453125, "pooled_tokens_per_turn": 240.17948717948718, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.18359375, "product_revisits": 0.0703125, "option_changes": 0.03125, "backtracks": 0.11328125 }, "terminations": { "purchase": 247, "generation_length": 9 } }, "success": { "n": 229, "tasks": 30, "metrics": { "model_steps": { "n": 229, "mean": 4.620087336244541, "median": 4.0, "p90": 6.0 }, "actions": { "n": 229, "mean": 4.458515283842795, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 229, "mean": 4.449781659388647, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 229, "mean": 217.06998683024887, "median": 204.0, "p90": 295.7500000000001 }, "generated_tokens": { "n": 229, "mean": 1004.9650655021834, "median": 875.0, "p90": 1394.8000000000002 }, "first_turn_tokens": { "n": 229, "mean": 162.0349344978166, "median": 159.0, "p90": 207.20000000000002 }, "searches": { "n": 229, "mean": 1.091703056768559, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 229, "mean": 1.091703056768559, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 229, "mean": 1.0829694323144106, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 229, "mean": 1.034934497816594, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 229, "mean": 0.048034934497816595, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 229, "mean": 0.09170305676855896, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 229, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 229, "mean": 1.1834061135371179, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 229, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 229, "mean": 0.1615720524017467, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 229, "mean": 0.008733624454148471, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 229, "mean": 284.17128647674497, "median": 265.6666666666667, "p90": 398.46999999999997 }, "first_product_rank": { "n": 229, "mean": 1.2096069868995634, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 217.52079395085067, "token_complete_episodes": 229, "rates": { "protocol_errors": 0.13973799126637554, "product_revisits": 0.039301310043668124, "option_changes": 0.0, "backtracks": 0.0611353711790393 }, "terminations": { "purchase": 229 } }, "failure": { "n": 27, "tasks": 6, "metrics": { "model_steps": { "n": 27, "mean": 9.925925925925926, "median": 7.0, "p90": 19.0 }, "actions": { "n": 27, "mean": 8.703703703703704, "median": 6.0, "p90": 16.0 }, "valid_actions": { "n": 27, "mean": 8.555555555555555, "median": 6.0, "p90": 16.0 }, "tokens_per_turn": { "n": 27, "mean": 323.3080001842429, "median": 250.0, "p90": 498.4333333333335 }, "generated_tokens": { "n": 27, "mean": 3271.8888888888887, "median": 3343.0, "p90": 6388.400000000001 }, "first_turn_tokens": { "n": 27, "mean": 126.81481481481481, "median": 112.0, "p90": 173.0 }, "searches": { "n": 27, "mean": 2.3333333333333335, "median": 2.0, "p90": 4.400000000000002 }, "unique_queries": { "n": 27, "mean": 2.2962962962962963, "median": 2.0, "p90": 4.400000000000002 }, "product_opens": { "n": 27, "mean": 2.037037037037037, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 27, "mean": 1.4814814814814814, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 27, "mean": 0.5555555555555556, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 27, "mean": 1.4074074074074074, "median": 1.0, "p90": 3.400000000000002 }, "pagination": { "n": 27, "mean": 0.2962962962962963, "median": 0.0, "p90": 0.40000000000000213 }, "option_selections": { "n": 27, "mean": 1.8148148148148149, "median": 1.0, "p90": 3.400000000000002 }, "option_changes": { "n": 27, "mean": 0.5185185185185185, "median": 0.0, "p90": 2.0 }, "protocol_errors": { "n": 27, "mean": 0.8888888888888888, "median": 1.0, "p90": 2.400000000000002 }, "invalid_actions": { "n": 27, "mean": 0.14814814814814814, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 27, "mean": 397.7095573881393, "median": 371.25, "p90": 566.7127819548873 }, "first_product_rank": { "n": 27, "mean": 2.962962962962963, "median": 2.0, "p90": 8.0 } }, "success_rate": 0.0, "reward_mean": 0.0, "pooled_tokens_per_turn": 329.6305970149254, "token_complete_episodes": 27, "rates": { "protocol_errors": 0.5555555555555556, "product_revisits": 0.3333333333333333, "option_changes": 0.2962962962962963, "backtracks": 0.5555555555555556 }, "terminations": { "generation_length": 9, "purchase": 18 } } }, { "step": 78, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.01953125, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.90625, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.73046875, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 199.68313851404673, "median": 188.0, "p90": 263.55 }, "generated_tokens": { "n": 256, "mean": 1044.18359375, "median": 771.0, "p90": 1525.0 }, "first_turn_tokens": { "n": 256, "mean": 155.5859375, "median": 151.0, "p90": 204.0 }, "searches": { "n": 256, "mean": 1.19140625, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.18359375, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.17578125, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0390625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.13671875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.19140625, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.19921875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0859375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.17578125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 259.3909829807779, "median": 235.725, "p90": 372.6857142857143 }, "first_product_rank": { "n": 256, "mean": 1.0546875, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.9140625, "reward_mean": 0.9296875, "pooled_tokens_per_turn": 208.0241245136187, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.07421875, "product_revisits": 0.078125, "option_changes": 0.00390625, "backtracks": 0.0859375 }, "terminations": { "purchase": 249, "generation_length": 7 } }, "success": { "n": 234, "tasks": 30, "metrics": { "model_steps": { "n": 234, "mean": 4.435897435897436, "median": 4.0, "p90": 5.0 }, "actions": { "n": 234, "mean": 4.367521367521367, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 234, "mean": 4.358974358974359, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 234, "mean": 197.20341847729665, "median": 184.25, "p90": 260.7125000000001 }, "generated_tokens": { "n": 234, "mean": 898.6025641025641, "median": 752.0, "p90": 1247.2 }, "first_turn_tokens": { "n": 234, "mean": 156.68803418803418, "median": 152.0, "p90": 206.70000000000002 }, "searches": { "n": 234, "mean": 1.064102564102564, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 234, "mean": 1.0598290598290598, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 234, "mean": 1.064102564102564, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 234, "mean": 1.0128205128205128, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 234, "mean": 0.05128205128205128, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 234, "mean": 0.0641025641025641, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 234, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 234, "mean": 1.1666666666666667, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 234, "mean": 0.008547008547008548, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 234, "mean": 0.06837606837606838, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 234, "mean": 0.008547008547008548, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 234, "mean": 253.91937186151932, "median": 226.25, "p90": 360.8450000000001 }, "first_product_rank": { "n": 234, "mean": 1.0598290598290598, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 202.57514450867052, "token_complete_episodes": 234, "rates": { "protocol_errors": 0.0641025641025641, "product_revisits": 0.038461538461538464, "option_changes": 0.004273504273504274, "backtracks": 0.04700854700854701 }, "terminations": { "purchase": 234 } }, "failure": { "n": 22, "tasks": 4, "metrics": { "model_steps": { "n": 22, "mean": 11.227272727272727, "median": 9.0, "p90": 20.900000000000002 }, "actions": { "n": 22, "mean": 10.636363636363637, "median": 8.5, "p90": 18.900000000000002 }, "valid_actions": { "n": 22, "mean": 8.681818181818182, "median": 6.5, "p90": 13.900000000000002 }, "tokens_per_turn": { "n": 22, "mean": 226.05834254129707, "median": 224.47500000000002, "p90": 266.05 }, "generated_tokens": { "n": 22, "mean": 2592.6363636363635, "median": 2320.0, "p90": 4537.1 }, "first_turn_tokens": { "n": 22, "mean": 143.86363636363637, "median": 145.5, "p90": 169.9 }, "searches": { "n": 22, "mean": 2.5454545454545454, "median": 1.5, "p90": 5.0 }, "unique_queries": { "n": 22, "mean": 2.5, "median": 1.5, "p90": 5.0 }, "product_opens": { "n": 22, "mean": 2.3636363636363638, "median": 1.5, "p90": 4.0 }, "unique_products": { "n": 22, "mean": 1.3181818181818181, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 22, "mean": 1.0454545454545454, "median": 0.5, "p90": 2.0 }, "backtracks": { "n": 22, "mean": 1.5454545454545454, "median": 0.5, "p90": 4.0 }, "pagination": { "n": 22, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 22, "mean": 1.5454545454545454, "median": 2.0, "p90": 2.0 }, "option_changes": { "n": 22, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 22, "mean": 0.2727272727272727, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 22, "mean": 1.9545454545454546, "median": 0.0, "p90": 5.900000000000002 }, "reasoning_chars_per_turn": { "n": 22, "mean": 317.5890285219831, "median": 322.425, "p90": 394.6538461538462 }, "first_product_rank": { "n": 22, "mean": 1.0, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.18181818181818182, "pooled_tokens_per_turn": 230.92307692307693, "token_complete_episodes": 22, "rates": { "protocol_errors": 0.18181818181818182, "product_revisits": 0.5, "option_changes": 0.0, "backtracks": 0.5 }, "terminations": { "purchase": 15, "generation_length": 7 } } }, { "step": 79, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 4.625, "median": 4.0, "p90": 5.0 }, "actions": { "n": 256, "mean": 4.55078125, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.51953125, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 192.76937713436737, "median": 181.75, "p90": 251.75 }, "generated_tokens": { "n": 256, "mean": 911.59765625, "median": 750.0, "p90": 1230.5 }, "first_turn_tokens": { "n": 256, "mean": 160.375, "median": 157.5, "p90": 216.0 }, "searches": { "n": 256, "mean": 1.0703125, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.0703125, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.0703125, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.0390625, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.0703125, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0078125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.3046875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.015625, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.0703125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 236.14831164386828, "median": 218.425, "p90": 324.625 }, "first_product_rank": { "n": 256, "mean": 1.07421875, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.90625, "reward_mean": 0.9085286458333334, "pooled_tokens_per_turn": 197.10219594594594, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.06640625, "product_revisits": 0.0234375, "option_changes": 0.015625, "backtracks": 0.05078125 }, "terminations": { "purchase": 255, "generation_length": 1 } }, "success": { "n": 232, "tasks": 31, "metrics": { "model_steps": { "n": 232, "mean": 4.4094827586206895, "median": 4.0, "p90": 5.0 }, "actions": { "n": 232, "mean": 4.366379310344827, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 232, "mean": 4.3405172413793105, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 232, "mean": 177.0168903414593, "median": 176.325, "p90": 230.95 }, "generated_tokens": { "n": 232, "mean": 786.7025862068965, "median": 735.5, "p90": 994.3000000000001 }, "first_turn_tokens": { "n": 232, "mean": 159.31465517241378, "median": 157.0, "p90": 208.9 }, "searches": { "n": 232, "mean": 1.0301724137931034, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 232, "mean": 1.0301724137931034, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 232, "mean": 1.0301724137931034, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 232, "mean": 1.021551724137931, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 232, "mean": 0.008620689655172414, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 232, "mean": 0.03017241379310345, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 232, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 232, "mean": 1.25, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 232, "mean": 0.01293103448275862, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 232, "mean": 0.04310344827586207, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 232, "mean": 0.02586206896551724, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 232, "mean": 218.8588563376279, "median": 212.0, "p90": 294.675 }, "first_product_rank": { "n": 232, "mean": 1.0603448275862069, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 178.41153470185728, "token_complete_episodes": 232, "rates": { "protocol_errors": 0.04310344827586207, "product_revisits": 0.008620689655172414, "option_changes": 0.01293103448275862, "backtracks": 0.02586206896551724 }, "terminations": { "purchase": 232 } }, "failure": { "n": 24, "tasks": 5, "metrics": { "model_steps": { "n": 24, "mean": 6.708333333333333, "median": 5.0, "p90": 12.0 }, "actions": { "n": 24, "mean": 6.333333333333333, "median": 5.0, "p90": 11.0 }, "valid_actions": { "n": 24, "mean": 6.25, "median": 5.0, "p90": 11.0 }, "tokens_per_turn": { "n": 24, "mean": 345.0434161324786, "median": 299.4666666666667, "p90": 455.59999999999997 }, "generated_tokens": { "n": 24, "mean": 2118.9166666666665, "median": 1565.0, "p90": 4519.0 }, "first_turn_tokens": { "n": 24, "mean": 170.625, "median": 170.5, "p90": 240.4 }, "searches": { "n": 24, "mean": 1.4583333333333333, "median": 1.0, "p90": 2.6999999999999993 }, "unique_queries": { "n": 24, "mean": 1.4583333333333333, "median": 1.0, "p90": 2.6999999999999993 }, "product_opens": { "n": 24, "mean": 1.4583333333333333, "median": 1.0, "p90": 2.6999999999999993 }, "unique_products": { "n": 24, "mean": 1.2083333333333333, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 24, "mean": 0.25, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 24, "mean": 0.4583333333333333, "median": 0.0, "p90": 1.6999999999999993 }, "pagination": { "n": 24, "mean": 0.08333333333333333, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 24, "mean": 1.8333333333333333, "median": 2.0, "p90": 2.0 }, "option_changes": { "n": 24, "mean": 0.041666666666666664, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 24, "mean": 0.3333333333333333, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 24, "mean": 0.08333333333333333, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 24, "mean": 403.2797129375254, "median": 398.79999999999995, "p90": 651.4292307692307 }, "first_product_rank": { "n": 24, "mean": 1.2083333333333333, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.024305555555555552, "pooled_tokens_per_turn": 315.8633540372671, "token_complete_episodes": 24, "rates": { "protocol_errors": 0.2916666666666667, "product_revisits": 0.16666666666666666, "option_changes": 0.041666666666666664, "backtracks": 0.2916666666666667 }, "terminations": { "generation_length": 1, "purchase": 23 } } }, { "step": 80, "all": { "n": 256, "tasks": 32, "metrics": { "model_steps": { "n": 256, "mean": 5.36328125, "median": 4.0, "p90": 5.5 }, "actions": { "n": 256, "mean": 5.2109375, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 256, "mean": 4.9921875, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 256, "mean": 202.13246904801315, "median": 176.125, "p90": 276.58888888888885 }, "generated_tokens": { "n": 256, "mean": 1151.9296875, "median": 745.0, "p90": 1930.0 }, "first_turn_tokens": { "n": 256, "mean": 152.3359375, "median": 151.0, "p90": 196.5 }, "searches": { "n": 256, "mean": 1.21875, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 256, "mean": 1.20703125, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 256, "mean": 1.1953125, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 256, "mean": 1.046875, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 256, "mean": 0.1484375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 256, "mean": 0.22265625, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 256, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 256, "mean": 1.39453125, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 256, "mean": 0.078125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 256, "mean": 0.11328125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 256, "mean": 0.21875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 256, "mean": 252.89486713740547, "median": 216.05, "p90": 379.725 }, "first_product_rank": { "n": 256, "mean": 1.19921875, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.9140625, "reward_mean": 0.921875, "pooled_tokens_per_turn": 214.78077203204663, "token_complete_episodes": 256, "rates": { "protocol_errors": 0.0859375, "product_revisits": 0.0859375, "option_changes": 0.06640625, "backtracks": 0.09375 }, "terminations": { "purchase": 246, "generation_length": 10 } }, "success": { "n": 234, "tasks": 32, "metrics": { "model_steps": { "n": 234, "mean": 4.585470085470085, "median": 4.0, "p90": 5.0 }, "actions": { "n": 234, "mean": 4.52991452991453, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 234, "mean": 4.517094017094017, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 234, "mean": 186.2453262380894, "median": 173.375, "p90": 251.785 }, "generated_tokens": { "n": 234, "mean": 873.1068376068376, "median": 717.5, "p90": 1269.5 }, "first_turn_tokens": { "n": 234, "mean": 152.43589743589743, "median": 150.5, "p90": 197.70000000000002 }, "searches": { "n": 234, "mean": 1.047008547008547, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 234, "mean": 1.047008547008547, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 234, "mean": 1.047008547008547, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 234, "mean": 1.017094017094017, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 234, "mean": 0.029914529914529916, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 234, "mean": 0.04700854700854701, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 234, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 234, "mean": 1.376068376068376, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 234, "mean": 0.0641025641025641, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 234, "mean": 0.05555555555555555, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 234, "mean": 0.01282051282051282, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 234, "mean": 234.80619589961697, "median": 210.1, "p90": 339.7150000000001 }, "first_product_rank": { "n": 234, "mean": 1.2179487179487178, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 190.40726933830382, "token_complete_episodes": 234, "rates": { "protocol_errors": 0.04700854700854701, "product_revisits": 0.02564102564102564, "option_changes": 0.0641025641025641, "backtracks": 0.03418803418803419 }, "terminations": { "purchase": 234 } }, "failure": { "n": 22, "tasks": 7, "metrics": { "model_steps": { "n": 22, "mean": 13.636363636363637, "median": 14.5, "p90": 21.800000000000004 }, "actions": { "n": 22, "mean": 12.454545454545455, "median": 13.0, "p90": 19.900000000000002 }, "valid_actions": { "n": 22, "mean": 10.045454545454545, "median": 11.5, "p90": 14.0 }, "tokens_per_turn": { "n": 22, "mean": 371.1138971172022, "median": 282.17149758454104, "p90": 599.4166666666667 }, "generated_tokens": { "n": 22, "mean": 4117.590909090909, "median": 4337.0, "p90": 6540.1 }, "first_turn_tokens": { "n": 22, "mean": 151.27272727272728, "median": 155.5, "p90": 191.8 }, "searches": { "n": 22, "mean": 3.0454545454545454, "median": 3.0, "p90": 5.0 }, "unique_queries": { "n": 22, "mean": 2.909090909090909, "median": 3.0, "p90": 5.0 }, "product_opens": { "n": 22, "mean": 2.772727272727273, "median": 3.0, "p90": 4.0 }, "unique_products": { "n": 22, "mean": 1.3636363636363635, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 22, "mean": 1.4090909090909092, "median": 1.0, "p90": 3.0 }, "backtracks": { "n": 22, "mean": 2.090909090909091, "median": 2.0, "p90": 4.0 }, "pagination": { "n": 22, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 22, "mean": 1.5909090909090908, "median": 1.5, "p90": 3.8000000000000043 }, "option_changes": { "n": 22, "mean": 0.22727272727272727, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 22, "mean": 0.7272727272727273, "median": 0.5, "p90": 2.0 }, "invalid_actions": { "n": 22, "mean": 2.409090909090909, "median": 1.0, "p90": 5.900000000000002 }, "reasoning_chars_per_turn": { "n": 22, "mean": 445.29255212115623, "median": 373.0599415204679, "p90": 805.3800000000003 }, "first_product_rank": { "n": 22, "mean": 1.0, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.09090909090909091, "pooled_tokens_per_turn": 301.95666666666665, "token_complete_episodes": 22, "rates": { "protocol_errors": 0.5, "product_revisits": 0.7272727272727273, "option_changes": 0.09090909090909091, "backtracks": 0.7272727272727273 }, "terminations": { "purchase": 12, "generation_length": 10 } } } ], "evaluation": { "val_base": { "n": 400, "tasks": 400, "metrics": { "model_steps": { "n": 400, "mean": 6.0075, "median": 4.0, "p90": 10.0 }, "actions": { "n": 400, "mean": 5.8125, "median": 4.0, "p90": 9.100000000000023 }, "valid_actions": { "n": 400, "mean": 5.41, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 400, "mean": 282.3884051441009, "median": 241.225, "p90": 379.45000000000005 }, "generated_tokens": { "n": 400, "mean": 1704.9275, "median": 1098.5, "p90": 4545.300000000004 }, "first_turn_tokens": { "n": 400, "mean": 216.6125, "median": 217.0, "p90": 284.0 }, "searches": { "n": 400, "mean": 1.2975, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 400, "mean": 1.285, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 400, "mean": 1.2475, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 400, "mean": 1.16, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 400, "mean": 0.0875, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 400, "mean": 0.305, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 400, "mean": 0.245, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 400, "mean": 1.385, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 400, "mean": 0.0475, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 400, "mean": 0.1475, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 400, "mean": 0.4025, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 400, "mean": 377.6462228492944, "median": 314.8666666666667, "p90": 504.66000000000014 }, "first_product_rank": { "n": 399, "mean": 4.957393483709273, "median": 2.0, "p90": 15.0 } }, "success_rate": 0.56, "reward_mean": 0.5804761904761905, "pooled_tokens_per_turn": 283.7998335414066, "token_complete_episodes": 400, "rates": { "protocol_errors": 0.0775, "product_revisits": 0.06, "option_changes": 0.04, "backtracks": 0.1425 }, "terminations": { "purchase": 372, "action_limit": 9, "generation_length": 19 } }, "val_20": { "n": 400, "tasks": 400, "metrics": { "model_steps": { "n": 400, "mean": 5.5775, "median": 4.0, "p90": 8.0 }, "actions": { "n": 400, "mean": 5.4325, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 400, "mean": 5.315, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 400, "mean": 219.90397907010149, "median": 185.775, "p90": 267.70086956521743 }, "generated_tokens": { "n": 400, "mean": 1233.9875, "median": 805.0, "p90": 2192.700000000007 }, "first_turn_tokens": { "n": 400, "mean": 167.405, "median": 160.5, "p90": 216.10000000000002 }, "searches": { "n": 400, "mean": 1.24, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 400, "mean": 1.225, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 400, "mean": 1.1825, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 400, "mean": 1.11, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 400, "mean": 0.0725, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 400, "mean": 0.26, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 400, "mean": 0.1875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 400, "mean": 1.485, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 400, "mean": 0.0875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 400, "mean": 0.1125, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 400, "mean": 0.1175, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 400, "mean": 283.3683977792787, "median": 229.25, "p90": 346.725 }, "first_product_rank": { "n": 398, "mean": 2.7537688442211055, "median": 1.0, "p90": 6.0 } }, "success_rate": 0.7475, "reward_mean": 0.765952380952381, "pooled_tokens_per_turn": 221.24383684446437, "token_complete_episodes": 400, "rates": { "protocol_errors": 0.0825, "product_revisits": 0.0475, "option_changes": 0.05, "backtracks": 0.1225 }, "terminations": { "purchase": 384, "action_limit": 3, "generation_length": 13 } }, "val_40": { "n": 400, "tasks": 400, "metrics": { "model_steps": { "n": 400, "mean": 5.9975, "median": 4.0, "p90": 9.0 }, "actions": { "n": 400, "mean": 5.86, "median": 4.0, "p90": 9.0 }, "valid_actions": { "n": 400, "mean": 5.645, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 400, "mean": 185.14573257530333, "median": 161.75, "p90": 238.41500000000005 }, "generated_tokens": { "n": 400, "mean": 1191.1325, "median": 696.5, "p90": 2268.3000000000025 }, "first_turn_tokens": { "n": 400, "mean": 131.765, "median": 130.0, "p90": 161.0 }, "searches": { "n": 400, "mean": 1.36, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 400, "mean": 1.35, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 400, "mean": 1.295, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 400, "mean": 1.08, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 400, "mean": 0.215, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 400, "mean": 0.3625, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 400, "mean": 0.0825, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 400, "mean": 1.5875, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 400, "mean": 0.0875, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 400, "mean": 0.115, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 400, "mean": 0.215, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 400, "mean": 232.0438372850674, "median": 192.225, "p90": 310.01176470588234 }, "first_product_rank": { "n": 400, "mean": 1.855, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.8, "reward_mean": 0.8131190476190477, "pooled_tokens_per_turn": 198.6048353480617, "token_complete_episodes": 400, "rates": { "protocol_errors": 0.0575, "product_revisits": 0.1275, "option_changes": 0.0475, "backtracks": 0.16 }, "terminations": { "purchase": 383, "generation_length": 9, "action_limit": 8 } }, "val_60": { "n": 400, "tasks": 400, "metrics": { "model_steps": { "n": 400, "mean": 5.7225, "median": 4.0, "p90": 9.0 }, "actions": { "n": 400, "mean": 5.515, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 400, "mean": 5.3075, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 400, "mean": 162.74313308743353, "median": 148.775, "p90": 217.34500000000003 }, "generated_tokens": { "n": 400, "mean": 1026.72, "median": 644.5, "p90": 1562.5000000000005 }, "first_turn_tokens": { "n": 400, "mean": 138.135, "median": 137.0, "p90": 173.0 }, "searches": { "n": 400, "mean": 1.2925, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 400, "mean": 1.285, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 400, "mean": 1.1925, "median": 1.0, "p90": 1.1000000000000227 }, "unique_products": { "n": 400, "mean": 1.03, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 400, "mean": 0.1625, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 400, "mean": 0.3, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 400, "mean": 0.09, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 400, "mean": 1.46, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 400, "mean": 0.0775, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 400, "mean": 0.2, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 400, "mean": 0.2075, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 400, "mean": 195.47809760243473, "median": 172.5, "p90": 279.6045454545455 }, "first_product_rank": { "n": 398, "mean": 1.6608040201005025, "median": 1.0, "p90": 2.0 } }, "success_rate": 0.8475, "reward_mean": 0.8601190476190476, "pooled_tokens_per_turn": 179.4180865006553, "token_complete_episodes": 400, "rates": { "protocol_errors": 0.085, "product_revisits": 0.0875, "option_changes": 0.0475, "backtracks": 0.1125 }, "terminations": { "purchase": 389, "action_limit": 8, "generation_length": 3 } }, "val_80": { "n": 400, "tasks": 400, "metrics": { "model_steps": { "n": 400, "mean": 5.555, "median": 4.0, "p90": 7.100000000000023 }, "actions": { "n": 400, "mean": 5.18, "median": 4.0, "p90": 7.0 }, "valid_actions": { "n": 400, "mean": 5.0925, "median": 4.0, "p90": 7.0 }, "tokens_per_turn": { "n": 400, "mean": 159.45544650660062, "median": 137.0, "p90": 198.92500000000004 }, "generated_tokens": { "n": 400, "mean": 1021.5, "median": 583.0, "p90": 1428.7000000000016 }, "first_turn_tokens": { "n": 400, "mean": 129.3675, "median": 128.0, "p90": 165.10000000000002 }, "searches": { "n": 400, "mean": 1.2575, "median": 1.0, "p90": 1.1000000000000227 }, "unique_queries": { "n": 400, "mean": 1.2525, "median": 1.0, "p90": 1.1000000000000227 }, "product_opens": { "n": 400, "mean": 1.21, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 400, "mean": 1.095, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 400, "mean": 0.115, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 400, "mean": 0.2625, "median": 0.0, "p90": 0.10000000000002274 }, "pagination": { "n": 400, "mean": 0.0175, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 400, "mean": 1.39, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 400, "mean": 0.0425, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 400, "mean": 0.3475, "median": 0.0, "p90": 0.10000000000002274 }, "invalid_actions": { "n": 400, "mean": 0.0875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 400, "mean": 193.30172975999113, "median": 155.0, "p90": 244.32545454545456 }, "first_product_rank": { "n": 400, "mean": 1.3, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.86, "reward_mean": 0.8694166666666666, "pooled_tokens_per_turn": 183.88838883888388, "token_complete_episodes": 400, "rates": { "protocol_errors": 0.1, "product_revisits": 0.06, "option_changes": 0.025, "backtracks": 0.1 }, "terminations": { "purchase": 382, "action_limit": 7, "generation_length": 11 } }, "test_base_free": { "n": 400, "tasks": 400, "metrics": { "model_steps": { "n": 400, "mean": 6.2125, "median": 4.0, "p90": 11.0 }, "actions": { "n": 400, "mean": 6.0225, "median": 4.0, "p90": 10.0 }, "valid_actions": { "n": 400, "mean": 5.4625, "median": 4.0, "p90": 9.0 }, "tokens_per_turn": { "n": 400, "mean": 295.88015047469867, "median": 241.875, "p90": 407.5366666666667 }, "generated_tokens": { "n": 400, "mean": 1780.4575, "median": 1145.0, "p90": 4473.100000000005 }, "first_turn_tokens": { "n": 400, "mean": 219.545, "median": 216.5, "p90": 293.0 }, "searches": { "n": 400, "mean": 1.325, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 400, "mean": 1.3175, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 400, "mean": 1.27, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 400, "mean": 1.165, "median": 1.0, "p90": 1.1000000000000227 }, "product_revisits": { "n": 400, "mean": 0.105, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 400, "mean": 0.3275, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 400, "mean": 0.1825, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 400, "mean": 1.425, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 400, "mean": 0.0575, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 400, "mean": 0.1425, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 400, "mean": 0.56, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 400, "mean": 398.26165468583133, "median": 308.7678571428571, "p90": 542.115 }, "first_product_rank": { "n": 397, "mean": 5.380352644836272, "median": 2.0, "p90": 14.0 } }, "success_rate": 0.5175, "reward_mean": 0.5421666666666667, "pooled_tokens_per_turn": 286.59275653923544, "token_complete_episodes": 400, "rates": { "protocol_errors": 0.0975, "product_revisits": 0.0675, "option_changes": 0.045, "backtracks": 0.15 }, "terminations": { "purchase": 373, "generation_length": 19, "action_limit": 8 } }, "test_base_s0": { "n": 400, "tasks": 400, "metrics": { "model_steps": { "n": 400, "mean": 6.6125, "median": 4.0, "p90": 12.0 }, "actions": { "n": 400, "mean": 6.34, "median": 4.0, "p90": 11.100000000000023 }, "valid_actions": { "n": 400, "mean": 5.69, "median": 4.0, "p90": 10.0 }, "tokens_per_turn": { "n": 400, "mean": 300.5632168917898, "median": 273.625, "p90": 419.57500000000005 }, "generated_tokens": { "n": 400, "mean": 1942.4375, "median": 1268.0, "p90": 3836.0 }, "first_turn_tokens": { "n": 400, "mean": 330.875, "median": 330.0, "p90": 409.0 }, "searches": { "n": 400, "mean": 1.4025, "median": 1.0, "p90": 2.1000000000000227 }, "unique_queries": { "n": 400, "mean": 1.3825, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 400, "mean": 1.305, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 400, "mean": 1.17, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 400, "mean": 0.135, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 400, "mean": 0.41, "median": 0.0, "p90": 1.1000000000000227 }, "pagination": { "n": 400, "mean": 0.265, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 400, "mean": 1.3675, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 400, "mean": 0.0325, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 400, "mean": 0.2475, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 400, "mean": 0.65, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 400, "mean": 398.3098028759367, "median": 363.75, "p90": 553.275 }, "first_product_rank": { "n": 399, "mean": 4.37593984962406, "median": 1.0, "p90": 12.0 } }, "success_rate": 0.545, "reward_mean": 0.5663012566137566, "pooled_tokens_per_turn": 293.7523629489603, "token_complete_episodes": 400, "rates": { "protocol_errors": 0.115, "product_revisits": 0.0825, "option_changes": 0.025, "backtracks": 0.2025 }, "terminations": { "purchase": 376, "generation_length": 10, "action_limit": 14 } }, "test_iter80_free": { "n": 400, "tasks": 400, "metrics": { "model_steps": { "n": 400, "mean": 5.7525, "median": 4.0, "p90": 8.0 }, "actions": { "n": 400, "mean": 5.36, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 400, "mean": 5.3175, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 400, "mean": 172.9325501156805, "median": 139.375, "p90": 217.90450980392157 }, "generated_tokens": { "n": 400, "mean": 1101.3225, "median": 600.0, "p90": 1943.6000000000035 }, "first_turn_tokens": { "n": 400, "mean": 130.96, "median": 130.0, "p90": 164.10000000000002 }, "searches": { "n": 400, "mean": 1.3225, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 400, "mean": 1.3225, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 400, "mean": 1.2725, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 400, "mean": 1.115, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 400, "mean": 0.1575, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 400, "mean": 0.33, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 400, "mean": 0.005, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 400, "mean": 1.44, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 400, "mean": 0.0675, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 400, "mean": 0.36, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 400, "mean": 0.0425, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 400, "mean": 215.63001722269357, "median": 157.125, "p90": 289.28235294117655 }, "first_product_rank": { "n": 400, "mean": 1.2775, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.8425, "reward_mean": 0.8500595238095238, "pooled_tokens_per_turn": 191.45110821382008, "token_complete_episodes": 400, "rates": { "protocol_errors": 0.105, "product_revisits": 0.0775, "option_changes": 0.045, "backtracks": 0.12 }, "terminations": { "purchase": 379, "action_limit": 8, "generation_length": 13 } }, "test_iter80_st": { "n": 400, "tasks": 400, "metrics": { "model_steps": { "n": 400, "mean": 6.01, "median": 4.0, "p90": 8.0 }, "actions": { "n": 400, "mean": 5.2775, "median": 4.0, "p90": 7.0 }, "valid_actions": { "n": 400, "mean": 5.22, "median": 4.0, "p90": 7.0 }, "tokens_per_turn": { "n": 400, "mean": 191.01577372825565, "median": 158.63333333333333, "p90": 241.7000000000001 }, "generated_tokens": { "n": 400, "mean": 1249.1075, "median": 680.0, "p90": 2702.1000000000004 }, "first_turn_tokens": { "n": 400, "mean": 167.865, "median": 161.0, "p90": 218.10000000000002 }, "searches": { "n": 400, "mean": 1.3325, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 400, "mean": 1.3225, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 400, "mean": 1.2375, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 400, "mean": 1.07, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 400, "mean": 0.1675, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 400, "mean": 0.3575, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 400, "mean": 0.0025, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 400, "mean": 1.36, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 400, "mean": 0.04, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 400, "mean": 0.69, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 400, "mean": 0.0575, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 400, "mean": 245.71354043765407, "median": 185.25, "p90": 321.53666666666686 }, "first_product_rank": { "n": 398, "mean": 1.185929648241206, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.8625, "reward_mean": 0.8671428571428572, "pooled_tokens_per_turn": 207.8381863560732, "token_complete_episodes": 400, "rates": { "protocol_errors": 0.1425, "product_revisits": 0.085, "option_changes": 0.0375, "backtracks": 0.115 }, "terminations": { "purchase": 372, "action_limit": 11, "generation_length": 17 } } }, "evaluation_outcomes": { "val_base": { "success": { "n": 224, "tasks": 224, "metrics": { "model_steps": { "n": 224, "mean": 4.866071428571429, "median": 4.0, "p90": 5.0 }, "actions": { "n": 224, "mean": 4.830357142857143, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 224, "mean": 4.75, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 224, "mean": 231.94718458748594, "median": 214.875, "p90": 323.875 }, "generated_tokens": { "n": 224, "mean": 1141.6964285714287, "median": 920.0, "p90": 1662.5000000000002 }, "first_turn_tokens": { "n": 224, "mean": 208.72767857142858, "median": 208.5, "p90": 272.0 }, "searches": { "n": 224, "mean": 1.1294642857142858, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 224, "mean": 1.1294642857142858, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 224, "mean": 1.1205357142857142, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 224, "mean": 1.0848214285714286, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 224, "mean": 0.03571428571428571, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 224, "mean": 0.12946428571428573, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 224, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 224, "mean": 1.3392857142857142, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 224, "mean": 0.049107142857142856, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 224, "mean": 0.03571428571428571, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 224, "mean": 0.08035714285714286, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 224, "mean": 296.58267884521234, "median": 272.375, "p90": 417.85 }, "first_product_rank": { "n": 224, "mean": 3.357142857142857, "median": 1.0, "p90": 9.700000000000017 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 234.62385321100916, "token_complete_episodes": 224, "rates": { "protocol_errors": 0.022321428571428572, "product_revisits": 0.03125, "option_changes": 0.04017857142857143, "backtracks": 0.0625 }, "terminations": { "purchase": 224 } }, "failure": { "n": 176, "tasks": 176, "metrics": { "model_steps": { "n": 176, "mean": 7.4602272727272725, "median": 4.0, "p90": 16.0 }, "actions": { "n": 176, "mean": 7.0625, "median": 4.0, "p90": 15.5 }, "valid_actions": { "n": 176, "mean": 6.25, "median": 4.0, "p90": 12.5 }, "tokens_per_turn": { "n": 176, "mean": 346.5863222161563, "median": 273.275, "p90": 552.925 }, "generated_tokens": { "n": 176, "mean": 2421.7670454545455, "median": 1365.0, "p90": 5944.5 }, "first_turn_tokens": { "n": 176, "mean": 226.64772727272728, "median": 225.5, "p90": 292.0 }, "searches": { "n": 176, "mean": 1.5113636363636365, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 176, "mean": 1.4829545454545454, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 176, "mean": 1.4090909090909092, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 176, "mean": 1.2556818181818181, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 176, "mean": 0.1534090909090909, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 176, "mean": 0.5284090909090909, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 176, "mean": 0.5170454545454546, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 176, "mean": 1.4431818181818181, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 176, "mean": 0.045454545454545456, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 176, "mean": 0.2897727272727273, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 176, "mean": 0.8125, "median": 0.0, "p90": 2.0 }, "reasoning_chars_per_turn": { "n": 176, "mean": 480.8180061272173, "median": 355.375, "p90": 693.8 }, "first_product_rank": { "n": 175, "mean": 7.005714285714285, "median": 3.0, "p90": 18.0 } }, "success_rate": 0.0, "reward_mean": 0.046536796536796536, "pooled_tokens_per_turn": 324.6237623762376, "token_complete_episodes": 176, "rates": { "protocol_errors": 0.14772727272727273, "product_revisits": 0.09659090909090909, "option_changes": 0.03977272727272727, "backtracks": 0.24431818181818182 }, "terminations": { "purchase": 148, "action_limit": 9, "generation_length": 19 } } }, "val_20": { "success": { "n": 299, "tasks": 299, "metrics": { "model_steps": { "n": 299, "mean": 4.829431438127091, "median": 4.0, "p90": 6.0 }, "actions": { "n": 299, "mean": 4.769230769230769, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 299, "mean": 4.739130434782608, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 299, "mean": 186.45335980444673, "median": 179.75, "p90": 244.06666666666666 }, "generated_tokens": { "n": 299, "mean": 914.1973244147157, "median": 764.0, "p90": 1285.4 }, "first_turn_tokens": { "n": 299, "mean": 167.1571906354515, "median": 160.0, "p90": 215.2 }, "searches": { "n": 299, "mean": 1.1036789297658862, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 299, "mean": 1.1036789297658862, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 299, "mean": 1.0836120401337792, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 299, "mean": 1.0501672240802675, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 299, "mean": 0.033444816053511704, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 299, "mean": 0.10367892976588629, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 299, "mean": 0.046822742474916385, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 299, "mean": 1.4013377926421404, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 299, "mean": 0.06688963210702341, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 299, "mean": 0.06020066889632107, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 299, "mean": 0.030100334448160536, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 299, "mean": 228.45297588884546, "median": 218.5, "p90": 312.25 }, "first_product_rank": { "n": 299, "mean": 2.0334448160535117, "median": 1.0, "p90": 4.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 189.2970914127424, "token_complete_episodes": 299, "rates": { "protocol_errors": 0.06020066889632107, "product_revisits": 0.030100334448160536, "option_changes": 0.033444816053511704, "backtracks": 0.07023411371237458 }, "terminations": { "purchase": 299 } }, "failure": { "n": 101, "tasks": 101, "metrics": { "model_steps": { "n": 101, "mean": 7.792079207920792, "median": 5.0, "p90": 17.0 }, "actions": { "n": 101, "mean": 7.396039603960396, "median": 5.0, "p90": 17.0 }, "valid_actions": { "n": 101, "mean": 7.01980198019802, "median": 5.0, "p90": 16.0 }, "tokens_per_turn": { "n": 101, "mean": 318.93105986644554, "median": 207.43333333333334, "p90": 738.25 }, "generated_tokens": { "n": 101, "mean": 2180.6930693069307, "median": 1081.0, "p90": 5449.0 }, "first_turn_tokens": { "n": 101, "mean": 168.13861386138615, "median": 162.0, "p90": 217.0 }, "searches": { "n": 101, "mean": 1.6435643564356435, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 101, "mean": 1.5841584158415842, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 101, "mean": 1.4752475247524752, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 101, "mean": 1.2871287128712872, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 101, "mean": 0.18811881188118812, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 101, "mean": 0.7227722772277227, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 101, "mean": 0.6039603960396039, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 101, "mean": 1.7326732673267327, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 101, "mean": 0.1485148514851485, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 101, "mean": 0.26732673267326734, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 101, "mean": 0.37623762376237624, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 101, "mean": 445.9397952568979, "median": 264.0, "p90": 1141.25 }, "first_product_rank": { "n": 99, "mean": 4.929292929292929, "median": 1.0, "p90": 11.800000000000011 } }, "success_rate": 0.0, "reward_mean": 0.07307873644507308, "pooled_tokens_per_turn": 279.8602287166455, "token_complete_episodes": 101, "rates": { "protocol_errors": 0.1485148514851485, "product_revisits": 0.09900990099009901, "option_changes": 0.09900990099009901, "backtracks": 0.27722772277227725 }, "terminations": { "purchase": 85, "action_limit": 3, "generation_length": 13 } } }, "val_40": { "success": { "n": 320, "tasks": 320, "metrics": { "model_steps": { "n": 320, "mean": 4.98125, "median": 4.0, "p90": 8.0 }, "actions": { "n": 320, "mean": 4.946875, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 320, "mean": 4.9, "median": 4.0, "p90": 8.0 }, "tokens_per_turn": { "n": 320, "mean": 163.09081171866328, "median": 155.625, "p90": 218.60000000000002 }, "generated_tokens": { "n": 320, "mean": 858.596875, "median": 653.5, "p90": 1292.5000000000014 }, "first_turn_tokens": { "n": 320, "mean": 131.35, "median": 130.0, "p90": 161.0 }, "searches": { "n": 320, "mean": 1.1625, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 320, "mean": 1.1625, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 320, "mean": 1.146875, "median": 1.0, "p90": 2.0 }, "unique_products": { "n": 320, "mean": 1.0375, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 320, "mean": 0.109375, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 320, "mean": 0.1625, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 320, "mean": 0.021875, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 320, "mean": 1.40625, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 320, "mean": 0.03125, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 320, "mean": 0.034375, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 320, "mean": 0.046875, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 320, "mean": 196.366671262765, "median": 180.875, "p90": 280.1700000000001 }, "first_product_rank": { "n": 320, "mean": 1.625, "median": 1.0, "p90": 2.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 172.36574654956084, "token_complete_episodes": 320, "rates": { "protocol_errors": 0.01875, "product_revisits": 0.08125, "option_changes": 0.025, "backtracks": 0.103125 }, "terminations": { "purchase": 320 } }, "failure": { "n": 80, "tasks": 80, "metrics": { "model_steps": { "n": 80, "mean": 10.0625, "median": 5.0, "p90": 27.300000000000026 }, "actions": { "n": 80, "mean": 9.5125, "median": 5.0, "p90": 25.0 }, "valid_actions": { "n": 80, "mean": 8.625, "median": 5.0, "p90": 22.10000000000001 }, "tokens_per_turn": { "n": 80, "mean": 273.36541600186365, "median": 200.61666666666667, "p90": 437.0532608695653 }, "generated_tokens": { "n": 80, "mean": 2521.275, "median": 1326.5, "p90": 6085.600000000001 }, "first_turn_tokens": { "n": 80, "mean": 133.425, "median": 131.0, "p90": 162.3 }, "searches": { "n": 80, "mean": 2.15, "median": 1.0, "p90": 5.1000000000000085 }, "unique_queries": { "n": 80, "mean": 2.1, "median": 1.0, "p90": 5.0 }, "product_opens": { "n": 80, "mean": 1.8875, "median": 1.0, "p90": 4.1000000000000085 }, "unique_products": { "n": 80, "mean": 1.25, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 80, "mean": 0.6375, "median": 0.0, "p90": 2.1000000000000085 }, "backtracks": { "n": 80, "mean": 1.1625, "median": 0.0, "p90": 4.1000000000000085 }, "pagination": { "n": 80, "mean": 0.325, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 80, "mean": 2.3125, "median": 1.0, "p90": 5.0 }, "option_changes": { "n": 80, "mean": 0.3125, "median": 0.0, "p90": 1.0 }, "protocol_errors": { "n": 80, "mean": 0.4375, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 80, "mean": 0.8875, "median": 0.0, "p90": 1.1000000000000085 }, "reasoning_chars_per_turn": { "n": 80, "mean": 374.7525013742769, "median": 245.89166666666665, "p90": 604.047035573123 }, "first_product_rank": { "n": 80, "mean": 2.775, "median": 1.0, "p90": 5.1000000000000085 } }, "success_rate": 0.0, "reward_mean": 0.0655952380952381, "pooled_tokens_per_turn": 250.5614906832298, "token_complete_episodes": 80, "rates": { "protocol_errors": 0.2125, "product_revisits": 0.3125, "option_changes": 0.1375, "backtracks": 0.3875 }, "terminations": { "purchase": 63, "generation_length": 9, "action_limit": 8 } } }, "val_60": { "success": { "n": 339, "tasks": 339, "metrics": { "model_steps": { "n": 339, "mean": 4.778761061946903, "median": 4.0, "p90": 5.0 }, "actions": { "n": 339, "mean": 4.71976401179941, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 339, "mean": 4.67551622418879, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 339, "mean": 153.03148328301026, "median": 143.5, "p90": 204.84999999999997 }, "generated_tokens": { "n": 339, "mean": 768.5604719764012, "median": 610.0, "p90": 932.1999999999999 }, "first_turn_tokens": { "n": 339, "mean": 137.6253687315634, "median": 137.0, "p90": 169.0 }, "searches": { "n": 339, "mean": 1.1091445427728615, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 339, "mean": 1.103244837758112, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 339, "mean": 1.0766961651917404, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 339, "mean": 1.008849557522124, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 339, "mean": 0.06784660766961652, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 339, "mean": 0.10914454277286136, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 339, "mean": 0.017699115044247787, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 339, "mean": 1.3628318584070795, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 339, "mean": 0.04424778761061947, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 339, "mean": 0.058997050147492625, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 339, "mean": 0.04424778761061947, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 339, "mean": 179.85376749393757, "median": 164.8, "p90": 255.69230769230768 }, "first_product_rank": { "n": 339, "mean": 1.5221238938053097, "median": 1.0, "p90": 1.1999999999999886 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 160.8283950617284, "token_complete_episodes": 339, "rates": { "protocol_errors": 0.035398230088495575, "product_revisits": 0.0471976401179941, "option_changes": 0.032448377581120944, "backtracks": 0.05309734513274336 }, "terminations": { "purchase": 339 } }, "failure": { "n": 61, "tasks": 61, "metrics": { "model_steps": { "n": 61, "mean": 10.967213114754099, "median": 5.0, "p90": 30.0 }, "actions": { "n": 61, "mean": 9.934426229508198, "median": 5.0, "p90": 25.0 }, "valid_actions": { "n": 61, "mean": 8.819672131147541, "median": 5.0, "p90": 19.0 }, "tokens_per_turn": { "n": 61, "mean": 216.714432820212, "median": 185.5, "p90": 266.44444444444446 }, "generated_tokens": { "n": 61, "mean": 2461.409836065574, "median": 1017.0, "p90": 5776.0 }, "first_turn_tokens": { "n": 61, "mean": 140.9672131147541, "median": 133.0, "p90": 181.0 }, "searches": { "n": 61, "mean": 2.3114754098360657, "median": 1.0, "p90": 6.0 }, "unique_queries": { "n": 61, "mean": 2.2950819672131146, "median": 1.0, "p90": 6.0 }, "product_opens": { "n": 61, "mean": 1.8360655737704918, "median": 1.0, "p90": 4.0 }, "unique_products": { "n": 61, "mean": 1.1475409836065573, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 61, "mean": 0.6885245901639344, "median": 0.0, "p90": 3.0 }, "backtracks": { "n": 61, "mean": 1.360655737704918, "median": 0.0, "p90": 5.0 }, "pagination": { "n": 61, "mean": 0.4918032786885246, "median": 0.0, "p90": 1.0 }, "option_selections": { "n": 61, "mean": 2.0, "median": 1.0, "p90": 4.0 }, "option_changes": { "n": 61, "mean": 0.26229508196721313, "median": 0.0, "p90": 1.0 }, "protocol_errors": { "n": 61, "mean": 0.9836065573770492, "median": 0.0, "p90": 2.0 }, "invalid_actions": { "n": 61, "mean": 1.1147540983606556, "median": 0.0, "p90": 2.0 }, "reasoning_chars_per_turn": { "n": 61, "mean": 282.30839115621427, "median": 229.5, "p90": 338.65 }, "first_product_rank": { "n": 59, "mean": 2.457627118644068, "median": 1.0, "p90": 6.400000000000006 } }, "success_rate": 0.0, "reward_mean": 0.08274785323965651, "pooled_tokens_per_turn": 224.43348281016443, "token_complete_episodes": 61, "rates": { "protocol_errors": 0.36065573770491804, "product_revisits": 0.3114754098360656, "option_changes": 0.13114754098360656, "backtracks": 0.4426229508196721 }, "terminations": { "action_limit": 8, "purchase": 50, "generation_length": 3 } } }, "val_80": { "success": { "n": 344, "tasks": 344, "metrics": { "model_steps": { "n": 344, "mean": 4.787790697674419, "median": 4.0, "p90": 5.0 }, "actions": { "n": 344, "mean": 4.665697674418604, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 344, "mean": 4.627906976744186, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 344, "mean": 140.81518000030044, "median": 134.35, "p90": 173.47499999999997 }, "generated_tokens": { "n": 344, "mean": 716.5058139534884, "median": 568.5, "p90": 791.4999999999995 }, "first_turn_tokens": { "n": 344, "mean": 129.92732558139534, "median": 128.5, "p90": 165.0 }, "searches": { "n": 344, "mean": 1.1046511627906976, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 344, "mean": 1.1046511627906976, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 344, "mean": 1.0959302325581395, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 344, "mean": 1.052325581395349, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 344, "mean": 0.0436046511627907, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 344, "mean": 0.10465116279069768, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 344, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 344, "mean": 1.322674418604651, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 344, "mean": 0.011627906976744186, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 344, "mean": 0.12209302325581395, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 344, "mean": 0.0377906976744186, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 344, "mean": 161.30225514255258, "median": 150.8, "p90": 208.69999999999996 }, "first_product_rank": { "n": 344, "mean": 1.2906976744186047, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 149.65270188221007, "token_complete_episodes": 344, "rates": { "protocol_errors": 0.05232558139534884, "product_revisits": 0.02616279069767442, "option_changes": 0.011627906976744186, "backtracks": 0.05813953488372093 }, "terminations": { "purchase": 344 } }, "failure": { "n": 56, "tasks": 56, "metrics": { "model_steps": { "n": 56, "mean": 10.267857142857142, "median": 5.0, "p90": 30.0 }, "actions": { "n": 56, "mean": 8.339285714285714, "median": 5.0, "p90": 19.0 }, "valid_actions": { "n": 56, "mean": 7.946428571428571, "median": 5.0, "p90": 17.0 }, "tokens_per_turn": { "n": 56, "mean": 273.9599407595873, "median": 187.90833333333333, "p90": 559.4022556390977 }, "generated_tokens": { "n": 56, "mean": 2895.035714285714, "median": 1111.5, "p90": 7260.5 }, "first_turn_tokens": { "n": 56, "mean": 125.92857142857143, "median": 125.0, "p90": 161.5 }, "searches": { "n": 56, "mean": 2.1964285714285716, "median": 1.0, "p90": 5.5 }, "unique_queries": { "n": 56, "mean": 2.1607142857142856, "median": 1.0, "p90": 5.5 }, "product_opens": { "n": 56, "mean": 1.9107142857142858, "median": 1.0, "p90": 5.0 }, "unique_products": { "n": 56, "mean": 1.3571428571428572, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 56, "mean": 0.5535714285714286, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 56, "mean": 1.2321428571428572, "median": 0.0, "p90": 4.5 }, "pagination": { "n": 56, "mean": 0.125, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 56, "mean": 1.8035714285714286, "median": 1.0, "p90": 3.5 }, "option_changes": { "n": 56, "mean": 0.23214285714285715, "median": 0.0, "p90": 0.5 }, "protocol_errors": { "n": 56, "mean": 1.7321428571428572, "median": 0.0, "p90": 6.5 }, "invalid_actions": { "n": 56, "mean": 0.39285714285714285, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 56, "mean": 389.8699309813992, "median": 236.32916666666665, "p90": 880.5695488721805 }, "first_product_rank": { "n": 56, "mean": 1.3571428571428572, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.06726190476190476, "pooled_tokens_per_turn": 281.95130434782607, "token_complete_episodes": 56, "rates": { "protocol_errors": 0.39285714285714285, "product_revisits": 0.26785714285714285, "option_changes": 0.10714285714285714, "backtracks": 0.35714285714285715 }, "terminations": { "action_limit": 7, "generation_length": 11, "purchase": 38 } } }, "test_base_free": { "success": { "n": 207, "tasks": 207, "metrics": { "model_steps": { "n": 207, "mean": 5.0, "median": 4.0, "p90": 6.0 }, "actions": { "n": 207, "mean": 4.932367149758454, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 207, "mean": 4.855072463768116, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 207, "mean": 232.1654656603661, "median": 216.25, "p90": 321.12 }, "generated_tokens": { "n": 207, "mean": 1169.8019323671497, "median": 949.0, "p90": 1605.6000000000001 }, "first_turn_tokens": { "n": 207, "mean": 216.15458937198068, "median": 212.0, "p90": 288.4 }, "searches": { "n": 207, "mean": 1.1304347826086956, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 207, "mean": 1.1207729468599035, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 207, "mean": 1.106280193236715, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 207, "mean": 1.0869565217391304, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 207, "mean": 0.01932367149758454, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 207, "mean": 0.13043478260869565, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 207, "mean": 0.10144927536231885, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 207, "mean": 1.3864734299516908, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 207, "mean": 0.03864734299516908, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 207, "mean": 0.06763285024154589, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 207, "mean": 0.07729468599033816, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 207, "mean": 295.93418998596616, "median": 276.25, "p90": 413.3 }, "first_product_rank": { "n": 207, "mean": 4.628019323671498, "median": 1.0, "p90": 10.800000000000011 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 233.96038647342996, "token_complete_episodes": 207, "rates": { "protocol_errors": 0.057971014492753624, "product_revisits": 0.01932367149758454, "option_changes": 0.033816425120772944, "backtracks": 0.06763285024154589 }, "terminations": { "purchase": 207 } }, "failure": { "n": 193, "tasks": 193, "metrics": { "model_steps": { "n": 193, "mean": 7.512953367875648, "median": 4.0, "p90": 17.0 }, "actions": { "n": 193, "mean": 7.191709844559585, "median": 4.0, "p90": 16.0 }, "valid_actions": { "n": 193, "mean": 6.1139896373057, "median": 4.0, "p90": 11.800000000000011 }, "tokens_per_turn": { "n": 193, "mean": 364.2166258973248, "median": 281.57142857142856, "p90": 486.91040723981905 }, "generated_tokens": { "n": 193, "mean": 2435.4093264248704, "median": 1503.0, "p90": 6047.6 }, "first_turn_tokens": { "n": 193, "mean": 223.18134715025906, "median": 220.0, "p90": 293.8 }, "searches": { "n": 193, "mean": 1.5336787564766838, "median": 1.0, "p90": 3.0 }, "unique_queries": { "n": 193, "mean": 1.528497409326425, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 193, "mean": 1.4455958549222798, "median": 1.0, "p90": 2.8000000000000114 }, "unique_products": { "n": 193, "mean": 1.2487046632124352, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 193, "mean": 0.19689119170984457, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 193, "mean": 0.538860103626943, "median": 0.0, "p90": 2.0 }, "pagination": { "n": 193, "mean": 0.2694300518134715, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 193, "mean": 1.4663212435233162, "median": 1.0, "p90": 3.0 }, "option_changes": { "n": 193, "mean": 0.07772020725388601, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 193, "mean": 0.22279792746113988, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 193, "mean": 1.077720207253886, "median": 0.0, "p90": 2.8000000000000114 }, "reasoning_chars_per_turn": { "n": 193, "mean": 508.0118370323187, "median": 366.6, "p90": 733.8525490196079 }, "first_product_rank": { "n": 190, "mean": 6.2, "median": 4.0, "p90": 16.0 } }, "success_rate": 0.0, "reward_mean": 0.051122625215889464, "pooled_tokens_per_turn": 324.16137931034484, "token_complete_episodes": 193, "rates": { "protocol_errors": 0.13989637305699482, "product_revisits": 0.11917098445595854, "option_changes": 0.05699481865284974, "backtracks": 0.23834196891191708 }, "terminations": { "purchase": 166, "generation_length": 19, "action_limit": 8 } } }, "test_base_s0": { "success": { "n": 218, "tasks": 218, "metrics": { "model_steps": { "n": 218, "mean": 5.1422018348623855, "median": 4.0, "p90": 8.0 }, "actions": { "n": 218, "mean": 5.009174311926605, "median": 4.0, "p90": 8.0 }, "valid_actions": { "n": 218, "mean": 4.954128440366972, "median": 4.0, "p90": 7.300000000000011 }, "tokens_per_turn": { "n": 218, "mean": 276.3602581350861, "median": 259.1, "p90": 378.2250000000001 }, "generated_tokens": { "n": 218, "mean": 1418.7385321100917, "median": 1143.0, "p90": 2127.5000000000005 }, "first_turn_tokens": { "n": 218, "mean": 328.9036697247706, "median": 325.0, "p90": 409.0 }, "searches": { "n": 218, "mean": 1.18348623853211, "median": 1.0, "p90": 2.0 }, "unique_queries": { "n": 218, "mean": 1.165137614678899, "median": 1.0, "p90": 2.0 }, "product_opens": { "n": 218, "mean": 1.128440366972477, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 218, "mean": 1.1009174311926606, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 218, "mean": 0.027522935779816515, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 218, "mean": 0.1834862385321101, "median": 0.0, "p90": 1.0 }, "pagination": { "n": 218, "mean": 0.12385321100917432, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 218, "mean": 1.334862385321101, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 218, "mean": 0.027522935779816515, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 218, "mean": 0.13302752293577982, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 218, "mean": 0.05504587155963303, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 218, "mean": 361.32187161080515, "median": 336.25, "p90": 518.02 }, "first_product_rank": { "n": 218, "mean": 3.220183486238532, "median": 1.0, "p90": 8.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 275.90098126672615, "token_complete_episodes": 218, "rates": { "protocol_errors": 0.06880733944954129, "product_revisits": 0.027522935779816515, "option_changes": 0.027522935779816515, "backtracks": 0.11467889908256881 }, "terminations": { "purchase": 218 } }, "failure": { "n": 182, "tasks": 182, "metrics": { "model_steps": { "n": 182, "mean": 8.373626373626374, "median": 4.0, "p90": 24.600000000000023 }, "actions": { "n": 182, "mean": 7.934065934065934, "median": 4.0, "p90": 21.0 }, "valid_actions": { "n": 182, "mean": 6.571428571428571, "median": 4.0, "p90": 12.900000000000006 }, "tokens_per_turn": { "n": 182, "mean": 329.5535740838854, "median": 293.225, "p90": 440.9913793103449 }, "generated_tokens": { "n": 182, "mean": 2569.7252747252746, "median": 1544.5, "p90": 6885.100000000001 }, "first_turn_tokens": { "n": 182, "mean": 333.2362637362637, "median": 340.0, "p90": 408.0 }, "searches": { "n": 182, "mean": 1.664835164835165, "median": 1.0, "p90": 3.9000000000000057 }, "unique_queries": { "n": 182, "mean": 1.6428571428571428, "median": 1.0, "p90": 3.0 }, "product_opens": { "n": 182, "mean": 1.5164835164835164, "median": 1.0, "p90": 3.0 }, "unique_products": { "n": 182, "mean": 1.2527472527472527, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 182, "mean": 0.26373626373626374, "median": 0.0, "p90": 1.0 }, "backtracks": { "n": 182, "mean": 0.6813186813186813, "median": 0.0, "p90": 3.0 }, "pagination": { "n": 182, "mean": 0.4340659340659341, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 182, "mean": 1.4065934065934067, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 182, "mean": 0.038461538461538464, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 182, "mean": 0.38461538461538464, "median": 0.0, "p90": 1.0 }, "invalid_actions": { "n": 182, "mean": 1.3626373626373627, "median": 0.0, "p90": 3.0 }, "reasoning_chars_per_turn": { "n": 182, "mean": 442.61402823746806, "median": 383.5, "p90": 622.125 }, "first_product_rank": { "n": 181, "mean": 5.767955801104972, "median": 3.0, "p90": 15.0 } }, "success_rate": 0.0, "reward_mean": 0.04681594860166289, "pooled_tokens_per_turn": 306.88320209973756, "token_complete_episodes": 182, "rates": { "protocol_errors": 0.17032967032967034, "product_revisits": 0.14835164835164835, "option_changes": 0.02197802197802198, "backtracks": 0.3076923076923077 }, "terminations": { "purchase": 158, "generation_length": 10, "action_limit": 14 } } }, "test_iter80_free": { "success": { "n": 337, "tasks": 337, "metrics": { "model_steps": { "n": 337, "mean": 4.881305637982196, "median": 4.0, "p90": 6.0 }, "actions": { "n": 337, "mean": 4.771513353115727, "median": 4.0, "p90": 6.0 }, "valid_actions": { "n": 337, "mean": 4.750741839762611, "median": 4.0, "p90": 6.0 }, "tokens_per_turn": { "n": 337, "mean": 145.75104017969886, "median": 135.5, "p90": 184.10000000000002 }, "generated_tokens": { "n": 337, "mean": 745.5311572700297, "median": 578.0, "p90": 954.8000000000001 }, "first_turn_tokens": { "n": 337, "mean": 130.77151335311572, "median": 130.0, "p90": 164.40000000000003 }, "searches": { "n": 337, "mean": 1.1275964391691395, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 337, "mean": 1.1275964391691395, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 337, "mean": 1.1157270029673592, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 337, "mean": 1.0415430267062316, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 337, "mean": 0.07418397626112759, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 337, "mean": 0.12759643916913946, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 337, "mean": 0.002967359050445104, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 337, "mean": 1.3768545994065282, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 337, "mean": 0.05637982195845697, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 337, "mean": 0.10979228486646884, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 337, "mean": 0.020771513353115726, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 337, "mean": 168.88112421639195, "median": 151.25, "p90": 228.25500000000005 }, "first_product_rank": { "n": 337, "mean": 1.2017804154302671, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 152.73191489361702, "token_complete_episodes": 337, "rates": { "protocol_errors": 0.05341246290801187, "product_revisits": 0.04451038575667656, "option_changes": 0.04154302670623145, "backtracks": 0.0712166172106825 }, "terminations": { "purchase": 337 } }, "failure": { "n": 63, "tasks": 63, "metrics": { "model_steps": { "n": 63, "mean": 10.412698412698413, "median": 5.0, "p90": 30.0 }, "actions": { "n": 63, "mean": 8.507936507936508, "median": 5.0, "p90": 21.60000000000001 }, "valid_actions": { "n": 63, "mean": 8.34920634920635, "median": 5.0, "p90": 20.800000000000004 }, "tokens_per_turn": { "n": 63, "mean": 318.33205564624916, "median": 195.13333333333333, "p90": 547.2512820512823 }, "generated_tokens": { "n": 63, "mean": 3004.5238095238096, "median": 1175.0, "p90": 7056.400000000001 }, "first_turn_tokens": { "n": 63, "mean": 131.96825396825398, "median": 132.0, "p90": 160.20000000000005 }, "searches": { "n": 63, "mean": 2.365079365079365, "median": 1.0, "p90": 6.0 }, "unique_queries": { "n": 63, "mean": 2.365079365079365, "median": 1.0, "p90": 6.0 }, "product_opens": { "n": 63, "mean": 2.111111111111111, "median": 1.0, "p90": 5.800000000000004 }, "unique_products": { "n": 63, "mean": 1.507936507936508, "median": 1.0, "p90": 3.0 }, "product_revisits": { "n": 63, "mean": 0.6031746031746031, "median": 0.0, "p90": 2.0 }, "backtracks": { "n": 63, "mean": 1.4126984126984128, "median": 0.0, "p90": 5.800000000000004 }, "pagination": { "n": 63, "mean": 0.015873015873015872, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 63, "mean": 1.7777777777777777, "median": 1.0, "p90": 3.8000000000000043 }, "option_changes": { "n": 63, "mean": 0.12698412698412698, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 63, "mean": 1.6984126984126984, "median": 0.0, "p90": 5.800000000000004 }, "invalid_actions": { "n": 63, "mean": 0.15873015873015872, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 63, "mean": 465.69949251037076, "median": 249.56666666666666, "p90": 863.0380952380957 }, "first_product_rank": { "n": 63, "mean": 1.6825396825396826, "median": 1.0, "p90": 3.0 } }, "success_rate": 0.0, "reward_mean": 0.04799697656840514, "pooled_tokens_per_turn": 288.5442073170732, "token_complete_episodes": 63, "rates": { "protocol_errors": 0.38095238095238093, "product_revisits": 0.25396825396825395, "option_changes": 0.06349206349206349, "backtracks": 0.38095238095238093 }, "terminations": { "purchase": 42, "action_limit": 8, "generation_length": 13 } } }, "test_iter80_st": { "success": { "n": 345, "tasks": 345, "metrics": { "model_steps": { "n": 345, "mean": 4.820289855072463, "median": 4.0, "p90": 5.0 }, "actions": { "n": 345, "mean": 4.646376811594203, "median": 4.0, "p90": 5.0 }, "valid_actions": { "n": 345, "mean": 4.620289855072464, "median": 4.0, "p90": 5.0 }, "tokens_per_turn": { "n": 345, "mean": 163.02682570539622, "median": 154.25, "p90": 205.25000000000006 }, "generated_tokens": { "n": 345, "mean": 800.0347826086957, "median": 651.0, "p90": 1052.600000000001 }, "first_turn_tokens": { "n": 345, "mean": 167.04057971014493, "median": 159.0, "p90": 217.20000000000005 }, "searches": { "n": 345, "mean": 1.098550724637681, "median": 1.0, "p90": 1.0 }, "unique_queries": { "n": 345, "mean": 1.098550724637681, "median": 1.0, "p90": 1.0 }, "product_opens": { "n": 345, "mean": 1.098550724637681, "median": 1.0, "p90": 1.0 }, "unique_products": { "n": 345, "mean": 1.03768115942029, "median": 1.0, "p90": 1.0 }, "product_revisits": { "n": 345, "mean": 0.06086956521739131, "median": 0.0, "p90": 0.0 }, "backtracks": { "n": 345, "mean": 0.10434782608695652, "median": 0.0, "p90": 0.0 }, "pagination": { "n": 345, "mean": 0.0, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 345, "mean": 1.318840579710145, "median": 1.0, "p90": 2.0 }, "option_changes": { "n": 345, "mean": 0.043478260869565216, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 345, "mean": 0.17391304347826086, "median": 0.0, "p90": 0.0 }, "invalid_actions": { "n": 345, "mean": 0.02608695652173913, "median": 0.0, "p90": 0.0 }, "reasoning_chars_per_turn": { "n": 345, "mean": 195.02170625843675, "median": 177.25, "p90": 261.6000000000001 }, "first_product_rank": { "n": 345, "mean": 1.1710144927536232, "median": 1.0, "p90": 1.0 } }, "success_rate": 1.0, "reward_mean": 1.0, "pooled_tokens_per_turn": 165.97233914612147, "token_complete_episodes": 345, "rates": { "protocol_errors": 0.07536231884057971, "product_revisits": 0.034782608695652174, "option_changes": 0.04057971014492753, "backtracks": 0.05217391304347826 }, "terminations": { "purchase": 345 } }, "failure": { "n": 55, "tasks": 55, "metrics": { "model_steps": { "n": 55, "mean": 13.472727272727273, "median": 7.0, "p90": 30.0 }, "actions": { "n": 55, "mean": 9.236363636363636, "median": 5.0, "p90": 20.6 }, "valid_actions": { "n": 55, "mean": 8.981818181818182, "median": 5.0, "p90": 19.6 }, "tokens_per_turn": { "n": 55, "mean": 366.5828113261923, "median": 222.75, "p90": 824.8666666666668 }, "generated_tokens": { "n": 55, "mean": 4066.018181818182, "median": 4548.0, "p90": 8144.200000000001 }, "first_turn_tokens": { "n": 55, "mean": 173.03636363636363, "median": 167.0, "p90": 220.8 }, "searches": { "n": 55, "mean": 2.8, "median": 1.0, "p90": 7.0 }, "unique_queries": { "n": 55, "mean": 2.727272727272727, "median": 1.0, "p90": 6.0 }, "product_opens": { "n": 55, "mean": 2.109090909090909, "median": 1.0, "p90": 5.0 }, "unique_products": { "n": 55, "mean": 1.2727272727272727, "median": 1.0, "p90": 2.0 }, "product_revisits": { "n": 55, "mean": 0.8363636363636363, "median": 0.0, "p90": 3.0 }, "backtracks": { "n": 55, "mean": 1.9454545454545455, "median": 1.0, "p90": 6.0 }, "pagination": { "n": 55, "mean": 0.01818181818181818, "median": 0.0, "p90": 0.0 }, "option_selections": { "n": 55, "mean": 1.6181818181818182, "median": 1.0, "p90": 3.6000000000000014 }, "option_changes": { "n": 55, "mean": 0.01818181818181818, "median": 0.0, "p90": 0.0 }, "protocol_errors": { "n": 55, "mean": 3.9272727272727272, "median": 1.0, "p90": 11.600000000000001 }, "invalid_actions": { "n": 55, "mean": 0.2545454545454545, "median": 0.0, "p90": 1.0 }, "reasoning_chars_per_turn": { "n": 55, "mean": 563.6895911981991, "median": 272.0, "p90": 1425.2857142857147 }, "first_product_rank": { "n": 53, "mean": 1.2830188679245282, "median": 1.0, "p90": 1.0 } }, "success_rate": 0.0, "reward_mean": 0.03376623376623377, "pooled_tokens_per_turn": 301.7962213225371, "token_complete_episodes": 55, "rates": { "protocol_errors": 0.5636363636363636, "product_revisits": 0.4, "option_changes": 0.01818181818181818, "backtracks": 0.509090909090909 }, "terminations": { "action_limit": 11, "purchase": 27, "generation_length": 17 } } } }, "paired": { "fixed_val_all": { "paired_tasks": 400, "metrics": { "model_steps": { "n": 400, "a_mean": 6.0075, "b_mean": 5.555, "difference": -0.4525, "ci95": [ -1.0825, 0.16 ] }, "actions": { "n": 400, "a_mean": 5.8125, "b_mean": 5.18, "difference": -0.6325, "ci95": [ -1.1325, -0.12743750000000093 ] }, "valid_actions": { "n": 400, "a_mean": 5.41, "b_mean": 5.0925, "difference": -0.3175, "ci95": [ -0.755, 0.11 ] }, "tokens_per_turn": { "n": 400, "a_mean": 282.3884051441009, "b_mean": 159.45544650660062, "difference": -122.93295863750026, "ci95": [ -142.1904541703105, -104.31220769825248 ] }, "generated_tokens": { "n": 400, "a_mean": 1704.9275, "b_mean": 1021.5, "difference": -683.4275, "ci95": [ -898.903, -466.37468750000005 ] }, "first_turn_tokens": { "n": 400, "a_mean": 216.6125, "b_mean": 129.3675, "difference": -87.245, "ci95": [ -93.447625, -81.1774375 ] }, "searches": { "n": 400, "a_mean": 1.2975, "b_mean": 1.2575, "difference": -0.04, "ci95": [ -0.16, 0.08 ] }, "unique_queries": { "n": 400, "a_mean": 1.285, "b_mean": 1.2525, "difference": -0.0325, "ci95": [ -0.1525, 0.09 ] }, "product_opens": { "n": 400, "a_mean": 1.2475, "b_mean": 1.21, "difference": -0.0375, "ci95": [ -0.1475, 0.0675 ] }, "unique_products": { "n": 400, "a_mean": 1.16, "b_mean": 1.095, "difference": -0.065, "ci95": [ -0.1375, 0.0025 ] }, "product_revisits": { "n": 400, "a_mean": 0.0875, "b_mean": 0.115, "difference": 0.0275, "ci95": [ -0.0275, 0.0875 ] }, "backtracks": { "n": 400, "a_mean": 0.305, "b_mean": 0.2625, "difference": -0.0425, "ci95": [ -0.1675, 0.0825 ] }, "pagination": { "n": 400, "a_mean": 0.245, "b_mean": 0.0175, "difference": -0.2275, "ci95": [ -0.3675, -0.11 ] }, "option_selections": { "n": 400, "a_mean": 1.385, "b_mean": 1.39, "difference": 0.005, "ci95": [ -0.0775, 0.0875 ] }, "option_changes": { "n": 400, "a_mean": 0.0475, "b_mean": 0.0425, "difference": -0.005, "ci95": [ -0.04, 0.0375 ] }, "protocol_errors": { "n": 400, "a_mean": 0.1475, "b_mean": 0.3475, "difference": 0.2, "ci95": [ 0.0675, 0.3475 ] }, "invalid_actions": { "n": 400, "a_mean": 0.4025, "b_mean": 0.0875, "difference": -0.315, "ci95": [ -0.5, -0.155 ] }, "reasoning_chars_per_turn": { "n": 400, "a_mean": 377.6462228492944, "b_mean": 193.30172975999113, "difference": -184.3444930893034, "ci95": [ -217.251682619745, -153.50208147466512 ] }, "first_product_rank": { "n": 399, "a_mean": 4.957393483709273, "b_mean": 1.300751879699248, "difference": -3.656641604010025, "ci95": [ -4.4110275689223055, -2.962406015037594 ] } } }, "fixed_val_both_success": { "paired_tasks": 211, "metrics": { "model_steps": { "n": 211, "a_mean": 4.881516587677726, "b_mean": 4.639810426540285, "difference": -0.24170616113744076, "ci95": [ -0.6966824644549763, 0.1990521327014218 ] }, "actions": { "n": 211, "a_mean": 4.843601895734597, "b_mean": 4.549763033175355, "difference": -0.2938388625592417, "ci95": [ -0.7061611374407583, 0.10426540284360189 ] }, "valid_actions": { "n": 211, "a_mean": 4.767772511848341, "b_mean": 4.492890995260663, "difference": -0.27488151658767773, "ci95": [ -0.6445497630331753, 0.07109004739336493 ] }, "tokens_per_turn": { "n": 211, "a_mean": 228.69455528805057, "b_mean": 136.37167845382615, "difference": -92.32287683422439, "ci95": [ -102.08407096961423, -83.7909692461027 ] }, "generated_tokens": { "n": 211, "a_mean": 1131.691943127962, "b_mean": 674.4502369668246, "difference": -457.2417061611374, "ci95": [ -601.9885071090048, -313.0930094786731 ] }, "first_turn_tokens": { "n": 211, "a_mean": 206.6398104265403, "b_mean": 128.82938388625593, "difference": -77.81042654028435, "ci95": [ -85.39336492890995, -70.12274881516588 ] }, "searches": { "n": 211, "a_mean": 1.132701421800948, "b_mean": 1.0758293838862558, "difference": -0.05687203791469194, "ci95": [ -0.16113744075829384, 0.037914691943127965 ] }, "unique_queries": { "n": 211, "a_mean": 1.132701421800948, "b_mean": 1.0758293838862558, "difference": -0.05687203791469194, "ci95": [ -0.16113744075829384, 0.04265402843601896 ] }, "product_opens": { "n": 211, "a_mean": 1.1232227488151658, "b_mean": 1.0710900473933649, "difference": -0.052132701421800945, "ci95": [ -0.14703791469194308, 0.037914691943127965 ] }, "unique_products": { "n": 211, "a_mean": 1.0900473933649288, "b_mean": 1.028436018957346, "difference": -0.061611374407582936, "ci95": [ -0.13744075829383887, 0.0 ] }, "product_revisits": { "n": 211, "a_mean": 0.03317535545023697, "b_mean": 0.04265402843601896, "difference": 0.009478672985781991, "ci95": [ -0.037914691943127965, 0.05687203791469194 ] }, "backtracks": { "n": 211, "a_mean": 0.13270142180094788, "b_mean": 0.07582938388625593, "difference": -0.05687203791469194, "ci95": [ -0.16113744075829384, 0.037914691943127965 ] }, "pagination": { "n": 211, "a_mean": 0.03317535545023697, "b_mean": 0.0, "difference": -0.03317535545023697, "ci95": [ -0.07582938388625593, -0.004739336492890996 ] }, "option_selections": { "n": 211, "a_mean": 1.3459715639810426, "b_mean": 1.2701421800947867, "difference": -0.07582938388625593, "ci95": [ -0.15639810426540285, -0.004739336492890996 ] }, "option_changes": { "n": 211, "a_mean": 0.04739336492890995, "b_mean": 0.009478672985781991, "difference": -0.037914691943127965, "ci95": [ -0.07109004739336493, -0.014218009478672985 ] }, "protocol_errors": { "n": 211, "a_mean": 0.037914691943127965, "b_mean": 0.09004739336492891, "difference": 0.052132701421800945, "ci95": [ -0.018957345971563982, 0.12796208530805686 ] }, "invalid_actions": { "n": 211, "a_mean": 0.07582938388625593, "b_mean": 0.05687203791469194, "difference": -0.018957345971563982, "ci95": [ -0.11848341232227488, 0.08530805687203792 ] }, "reasoning_chars_per_turn": { "n": 211, "a_mean": 291.43378745135857, "b_mean": 155.44185767283614, "difference": -135.99192977852243, "ci95": [ -149.1483474229598, -123.33326590498267 ] }, "first_product_rank": { "n": 211, "a_mean": 3.308056872037915, "b_mean": 1.146919431279621, "difference": -2.161137440758294, "ci95": [ -2.9526066350710902, -1.4881516587677726 ] } } }, "train_common_tasks_r0_r3": { "paired_tasks": 0, "metrics": {} } }, "phases": [ { "round": 0, "phase": "search_home", "n": 6019, "mean": 194.06911447084232, "median": 185.0, "p90": 286.0 }, { "round": 0, "phase": "search_results", "n": 6650, "mean": 449.11834586466165, "median": 388.0, "p90": 830.0 }, { "round": 0, "phase": "product_detail", "n": 13772, "mean": 219.12990124891084, "median": 182.0, "p90": 389.89999999999964 }, { "round": 0, "phase": "protocol_repair", "n": 604, "mean": 255.43046357615893, "median": 211.0, "p90": 448.4000000000001 }, { "round": 1, "phase": "search_home", "n": 6380, "mean": 188.34655172413792, "median": 183.0, "p90": 255.0 }, { "round": 1, "phase": "search_results", "n": 6631, "mean": 346.9819031820238, "median": 331.0, "p90": 492.0 }, { "round": 1, "phase": "product_detail", "n": 14569, "mean": 242.8295009952639, "median": 214.0, "p90": 400.2000000000007 }, { "round": 1, "phase": "protocol_repair", "n": 657, "mean": 274.1674277016743, "median": 211.0, "p90": 464.0 }, { "round": 2, "phase": "search_home", "n": 6738, "mean": 140.8970020777679, "median": 132.0, "p90": 198.0 }, { "round": 2, "phase": "search_results", "n": 7016, "mean": 249.50855188141392, "median": 242.0, "p90": 352.0 }, { "round": 2, "phase": "product_detail", "n": 15270, "mean": 191.95081859855927, "median": 161.0, "p90": 329.0 }, { "round": 2, "phase": "protocol_repair", "n": 291, "mean": 230.83505154639175, "median": 216.0, "p90": 360.0 }, { "round": 3, "phase": "search_home", "n": 6143, "mean": 180.15155461500896, "median": 169.0, "p90": 242.0 }, { "round": 3, "phase": "search_results", "n": 6380, "mean": 295.4164576802508, "median": 279.0, "p90": 421.0 }, { "round": 3, "phase": "product_detail", "n": 13093, "mean": 215.77148094401588, "median": 166.0, "p90": 375.0 }, { "round": 3, "phase": "protocol_repair", "n": 587, "mean": 296.984667802385, "median": 273.0, "p90": 459.4 } ], "distributions": [ { "round": 0, "outcome": "success", "metric": "model_steps", "counts": { "4": 2331, "5": 700, "6": 30, "7": 20, "8": 56, "9": 37, "10": 14, "11": 8, "12": 16, "13": 4, "14": 7, "15": 3, "16": 3, "19": 2, "21": 1, "22": 1, "24": 1 } }, { "round": 0, "outcome": "success", "metric": "searches", "counts": { "1": 3078, "2": 113, "3": 36, "4": 2, "5": 4, "6": 1 } }, { "round": 0, "outcome": "success", "metric": "unique_products", "counts": { "1": 3101, "2": 106, "3": 22, "4": 2, "5": 2, "6": 1 } }, { "round": 0, "outcome": "success", "metric": "protocol_errors", "counts": { "0": 3151, "1": 68, "2": 14, "4": 1 } }, { "round": 0, "outcome": "failure", "metric": "model_steps", "counts": { "2": 8, "4": 931, "5": 328, "6": 65, "7": 41, "8": 77, "9": 51, "10": 36, "11": 42, "12": 42, "13": 29, "14": 18, "15": 14, "16": 21, "17": 16, "18": 15, "19": 18, "20": 12, "21": 6, "22": 8, "23": 9, "24": 8, "25": 5, "26": 5, "27": 2, "28": 7, "29": 1, "30": 12 } }, { "round": 0, "outcome": "failure", "metric": "searches", "counts": { "1": 1411, "2": 246, "3": 78, "4": 44, "5": 32, "6": 15, "7": 1 } }, { "round": 0, "outcome": "failure", "metric": "unique_products", "counts": { "0": 60, "1": 1430, "2": 217, "3": 78, "4": 29, "5": 10, "6": 3 } }, { "round": 0, "outcome": "failure", "metric": "protocol_errors", "counts": { "0": 1621, "1": 146, "2": 33, "3": 12, "4": 2, "5": 3, "6": 2, "18": 1, "29": 7 } }, { "round": 1, "outcome": "success", "metric": "model_steps", "counts": { "4": 3038, "5": 811, "6": 85, "7": 34, "8": 85, "9": 42, "10": 24, "11": 21, "12": 19, "13": 10, "14": 7, "15": 2, "16": 6, "17": 2, "18": 1, "19": 1, "21": 1, "22": 1, "23": 1 } }, { "round": 1, "outcome": "success", "metric": "searches", "counts": { "1": 3963, "2": 171, "3": 45, "4": 10, "5": 2 } }, { "round": 1, "outcome": "success", "metric": "unique_products", "counts": { "1": 4022, "2": 144, "3": 23, "4": 2 } }, { "round": 1, "outcome": "success", "metric": "protocol_errors", "counts": { "0": 4118, "1": 63, "2": 8, "3": 1, "5": 1 } }, { "round": 1, "outcome": "failure", "metric": "model_steps", "counts": { "4": 375, "5": 126, "6": 14, "7": 6, "8": 58, "9": 24, "10": 17, "11": 14, "12": 23, "13": 17, "14": 15, "15": 18, "16": 28, "17": 10, "18": 22, "19": 21, "20": 23, "21": 18, "22": 21, "23": 18, "24": 8, "25": 15, "26": 4, "27": 3, "28": 3, "29": 6, "30": 22 } }, { "round": 1, "outcome": "failure", "metric": "searches", "counts": { "1": 546, "2": 119, "3": 78, "4": 92, "5": 64, "6": 30 } }, { "round": 1, "outcome": "failure", "metric": "unique_products", "counts": { "0": 26, "1": 627, "2": 176, "3": 72, "4": 22, "5": 5, "6": 1 } }, { "round": 1, "outcome": "failure", "metric": "protocol_errors", "counts": { "0": 796, "1": 86, "2": 23, "3": 8, "4": 2, "29": 14 } }, { "round": 2, "outcome": "success", "metric": "model_steps", "counts": { "4": 3001, "5": 777, "6": 59, "7": 34, "8": 187, "9": 48, "10": 47, "11": 20, "12": 29, "13": 23, "14": 9, "15": 8, "16": 11, "17": 12, "18": 5, "19": 3, "20": 3, "21": 2, "22": 3 } }, { "round": 2, "outcome": "success", "metric": "searches", "counts": { "1": 3883, "2": 295, "3": 73, "4": 24, "5": 6 } }, { "round": 2, "outcome": "success", "metric": "unique_products", "counts": { "1": 4121, "2": 136, "3": 18, "4": 6 } }, { "round": 2, "outcome": "success", "metric": "protocol_errors", "counts": { "0": 4177, "1": 92, "2": 11, "3": 1 } }, { "round": 2, "outcome": "failure", "metric": "model_steps", "counts": { "4": 294, "5": 95, "6": 19, "7": 13, "8": 53, "9": 17, "10": 26, "11": 22, "12": 25, "13": 17, "14": 13, "15": 22, "16": 24, "17": 24, "18": 28, "19": 27, "20": 20, "21": 11, "22": 21, "23": 15, "24": 8, "25": 9, "26": 12, "27": 7, "28": 5, "29": 5, "30": 7 } }, { "round": 2, "outcome": "failure", "metric": "searches", "counts": { "1": 428, "2": 121, "3": 86, "4": 75, "5": 93, "6": 32, "7": 4 } }, { "round": 2, "outcome": "failure", "metric": "unique_products", "counts": { "0": 9, "1": 587, "2": 147, "3": 66, "4": 21, "5": 7, "6": 2 } }, { "round": 2, "outcome": "failure", "metric": "protocol_errors", "counts": { "0": 704, "1": 101, "2": 30, "3": 3, "4": 1 } }, { "round": 3, "outcome": "success", "metric": "model_steps", "counts": { "4": 3271, "5": 884, "6": 73, "7": 20, "8": 79, "9": 44, "10": 24, "11": 16, "12": 21, "13": 13, "14": 4, "15": 7, "16": 2, "17": 4, "18": 3, "19": 2, "20": 3, "22": 1 } }, { "round": 3, "outcome": "success", "metric": "searches", "counts": { "1": 4236, "2": 170, "3": 47, "4": 14, "5": 4 } }, { "round": 3, "outcome": "success", "metric": "unique_products", "counts": { "1": 4331, "2": 119, "3": 17, "4": 4 } }, { "round": 3, "outcome": "success", "metric": "protocol_errors", "counts": { "0": 4196, "1": 253, "2": 15, "3": 6, "4": 1 } }, { "round": 3, "outcome": "failure", "metric": "model_steps", "counts": { "2": 1, "3": 6, "4": 199, "5": 132, "6": 19, "7": 10, "8": 30, "9": 23, "10": 11, "11": 15, "12": 25, "13": 20, "14": 18, "15": 12, "16": 13, "17": 21, "18": 29, "19": 18, "20": 19, "21": 5, "22": 8, "23": 7, "24": 6, "25": 1, "27": 1 } }, { "round": 3, "outcome": "failure", "metric": "searches", "counts": { "1": 371, "2": 84, "3": 61, "4": 64, "5": 51, "6": 16, "7": 2 } }, { "round": 3, "outcome": "failure", "metric": "unique_products", "counts": { "0": 17, "1": 476, "2": 110, "3": 36, "4": 10 } }, { "round": 3, "outcome": "failure", "metric": "protocol_errors", "counts": { "0": 467, "1": 111, "2": 54, "3": 9, "4": 4, "5": 4 } } ], "cases": [ { "selection_rule": "largest_step_reduction_both_success", "task_id": 1828, "conditions": { "val_base": { "metrics": { "cohort": "val_base", "round": -1, "rollout_step": -1, "group_index": -1, "episode_id": "task-001828__sample-000", "task_id": 1828, "sample_id": 0, "mode": "evaluation", "scored": 1, "success": 1, "reward": 1.0, "outcome": "success", "termination": "purchase", "model_steps": 25, "actions": 25, "valid_actions": 24, "searches": 6, "unique_queries": 6, "product_opens": 6, "unique_products": 5, "product_revisits": 1, "backtracks": 5, "pagination": 0, "option_selections": 6, "option_changes": 0, "protocol_errors": 0, "invalid_actions": 1, "generated_tokens": 6174, "token_turns": 25, "tokens_per_turn": 246.96, "first_turn_tokens": 251, "raw_chars": 0, "raw_turns": 0, "reasoning_chars": 7822, "reasoning_chars_per_turn": 312.88, "first_product_rank": 7, "exposed_products": 73, "query_chars": 61, "token_basis": "api_completion_tokens", "search_home_turns": 6, "search_home_tokens": 1109, "search_results_turns": 6, "search_results_tokens": 2072, "product_detail_turns": 13, "product_detail_tokens": 2993, "other_turns": 0, "other_tokens": 0, "protocol_repair_turns": 0, "protocol_repair_tokens": 0 }, "actions": [ "search", "open_product", "invalid_action", "select_option", "backtrack", "search", "open_product", "select_option", "backtrack", "search", "open_product", "select_option", "backtrack", "search", "open_product", "select_option", "backtrack", "search", "open_product", "select_option", "backtrack", "search", "open_product", "select_option", "purchase" ] }, "val_80": { "metrics": { "cohort": "val_80", "round": -1, "rollout_step": -1, "group_index": -1, "episode_id": "task-001828__sample-000", "task_id": 1828, "sample_id": 0, "mode": "evaluation", "scored": 1, "success": 1, "reward": 1.0, "outcome": "success", "termination": "purchase", "model_steps": 4, "actions": 4, "valid_actions": 4, "searches": 1, "unique_queries": 1, "product_opens": 1, "unique_products": 1, "product_revisits": 0, "backtracks": 0, "pagination": 0, "option_selections": 1, "option_changes": 0, "protocol_errors": 0, "invalid_actions": 0, "generated_tokens": 740, "token_turns": 4, "tokens_per_turn": 185.0, "first_turn_tokens": 129, "raw_chars": 0, "raw_turns": 0, "reasoning_chars": 917, "reasoning_chars_per_turn": 229.25, "first_product_rank": 1, "exposed_products": 20, "query_chars": 30, "token_basis": "api_completion_tokens", "search_home_turns": 1, "search_home_tokens": 129, "search_results_turns": 1, "search_results_tokens": 215, "product_detail_turns": 2, "product_detail_tokens": 396, "other_turns": 0, "other_tokens": 0, "protocol_repair_turns": 0, "protocol_repair_tokens": 0 }, "actions": [ "search", "open_product", "select_option", "purchase" ] } } }, { "selection_rule": "failure_to_success_largest_token_reduction", "task_id": 3538, "conditions": { "val_base": { "metrics": { "cohort": "val_base", "round": -1, "rollout_step": -1, "group_index": -1, "episode_id": "task-003538__sample-000", "task_id": 3538, "sample_id": 0, "mode": "evaluation", "scored": 1, "success": 0, "reward": 0.0, "outcome": "failure", "termination": "action_limit", "model_steps": 30, "actions": 29, "valid_actions": 8, "searches": 3, "unique_queries": 3, "product_opens": 3, "unique_products": 2, "product_revisits": 1, "backtracks": 2, "pagination": 0, "option_selections": 0, "option_changes": 0, "protocol_errors": 1, "invalid_actions": 21, "generated_tokens": 13465, "token_turns": 30, "tokens_per_turn": 448.8333333333333, "first_turn_tokens": 217, "raw_chars": 0, "raw_turns": 0, "reasoning_chars": 18092, "reasoning_chars_per_turn": 603.0666666666667, "first_product_rank": 14, "exposed_products": 36, "query_chars": 50, "token_basis": "api_completion_tokens", "search_home_turns": 3, "search_home_tokens": 733, "search_results_turns": 3, "search_results_tokens": 1068, "product_detail_turns": 23, "product_detail_tokens": 11335, "other_turns": 0, "other_tokens": 0, "protocol_repair_turns": 1, "protocol_repair_tokens": 329 }, "actions": [ "search", "open_product", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "backtrack", "search", "open_product", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "invalid_action", "backtrack", "repair", "search", "open_product", "invalid_action", "invalid_action", "invalid_action", "invalid_action" ] }, "val_80": { "metrics": { "cohort": "val_80", "round": -1, "rollout_step": -1, "group_index": -1, "episode_id": "task-003538__sample-000", "task_id": 3538, "sample_id": 0, "mode": "evaluation", "scored": 1, "success": 1, "reward": 1.0, "outcome": "success", "termination": "purchase", "model_steps": 4, "actions": 4, "valid_actions": 4, "searches": 1, "unique_queries": 1, "product_opens": 1, "unique_products": 1, "product_revisits": 0, "backtracks": 0, "pagination": 0, "option_selections": 1, "option_changes": 0, "protocol_errors": 0, "invalid_actions": 0, "generated_tokens": 1045, "token_turns": 4, "tokens_per_turn": 261.25, "first_turn_tokens": 186, "raw_chars": 0, "raw_turns": 0, "reasoning_chars": 1291, "reasoning_chars_per_turn": 322.75, "first_product_rank": 5, "exposed_products": 20, "query_chars": 35, "token_basis": "api_completion_tokens", "search_home_turns": 1, "search_home_tokens": 186, "search_results_turns": 1, "search_results_tokens": 309, "product_detail_turns": 2, "product_detail_tokens": 550, "other_turns": 0, "other_tokens": 0, "protocol_repair_turns": 0, "protocol_repair_tokens": 0 }, "actions": [ "search", "open_product", "select_option", "purchase" ] } } }, { "selection_rule": "success_to_failure_longest_final", "task_id": 4359, "conditions": { "val_base": { "metrics": { "cohort": "val_base", "round": -1, "rollout_step": -1, "group_index": -1, "episode_id": "task-004359__sample-000", "task_id": 4359, "sample_id": 0, "mode": "evaluation", "scored": 1, "success": 1, "reward": 1.0, "outcome": "success", "termination": "purchase", "model_steps": 4, "actions": 4, "valid_actions": 4, "searches": 1, "unique_queries": 1, "product_opens": 1, "unique_products": 1, "product_revisits": 0, "backtracks": 0, "pagination": 0, "option_selections": 1, "option_changes": 0, "protocol_errors": 0, "invalid_actions": 0, "generated_tokens": 959, "token_turns": 4, "tokens_per_turn": 239.75, "first_turn_tokens": 172, "raw_chars": 0, "raw_turns": 0, "reasoning_chars": 1264, "reasoning_chars_per_turn": 316.0, "first_product_rank": 1, "exposed_products": 20, "query_chars": 14, "token_basis": "api_completion_tokens", "search_home_turns": 1, "search_home_tokens": 172, "search_results_turns": 1, "search_results_tokens": 235, "product_detail_turns": 2, "product_detail_tokens": 552, "other_turns": 0, "other_tokens": 0, "protocol_repair_turns": 0, "protocol_repair_tokens": 0 }, "actions": [ "search", "open_product", "select_option", "purchase" ] }, "val_80": { "metrics": { "cohort": "val_80", "round": -1, "rollout_step": -1, "group_index": -1, "episode_id": "task-004359__sample-000", "task_id": 4359, "sample_id": 0, "mode": "evaluation", "scored": 1, "success": 0, "reward": 0.0, "outcome": "failure", "termination": "action_limit", "model_steps": 30, "actions": 18, "valid_actions": 17, "searches": 7, "unique_queries": 7, "product_opens": 2, "unique_products": 1, "product_revisits": 1, "backtracks": 6, "pagination": 2, "option_selections": 0, "option_changes": 0, "protocol_errors": 12, "invalid_actions": 1, "generated_tokens": 6084, "token_turns": 30, "tokens_per_turn": 202.8, "first_turn_tokens": 131, "raw_chars": 0, "raw_turns": 0, "reasoning_chars": 7483, "reasoning_chars_per_turn": 249.43333333333334, "first_product_rank": 1, "exposed_products": 49, "query_chars": 229, "token_basis": "api_completion_tokens", "search_home_turns": 7, "search_home_tokens": 1251, "search_results_turns": 9, "search_results_tokens": 1852, "product_detail_turns": 2, "product_detail_tokens": 393, "other_turns": 0, "other_tokens": 0, "protocol_repair_turns": 12, "protocol_repair_tokens": 2588 }, "actions": [ "search", "open_product", "backtrack", "search", "repair", "paginate", "invalid_action", "repair", "backtrack", "search", "repair", "backtrack", "search", "open_product", "repair", "backtrack", "search", "paginate", "repair", "repair", "backtrack", "search", "repair", "repair", "repair", "repair", "backtrack", "search", "repair", "repair" ] } } } ], "uncertainty": { "method": "paired task bootstrap, 5000 resamples, numpy.default_rng, metric-hashed seed 20260909", "train_pairing": "average siblings and repeated episodes within each task/round, then pair common tasks", "success_conditioning": "descriptive/post-treatment; residual failure mix changes during learning" } }