{"experiment":"mcp-3-sol-v1","dataset_digest":"sha256:8a786725d3648a76282fc5868df06da3d5e00d603f283573a53430803465d78a","cohort_sha256":"8da119011fd0c6f69c188991f591bdd6e89526dc52e70121f655184f9d1f0611","rows":[{"task":"freecad-f32bdb2966","cell":"claude-code-opus--freecad-mcp","model":"anthropic/claude-opus-5-5","agent":"claude-code","reasoning_effort":"high","effort_launch_confirmed":true,"effort_evidence":{"source":"native launch log","option":"--effort high"},"observed_model":"claude-opus-5-5","observed_agent_version":"2.1.289","status":"partial","reward":0.999873,"geometry":0.9997457545508739,"spec":1.0,"details":{"reward":0.999873,"geometry_similarity":0.9997457545508739,"cad_spec_consistency":1.0,"combined":0.9998728611131953,"geometry_similarity_reason":"reference.FCStd vs answer.FCStd: overall=1.000 [solid_count=1 (matched)] (surface_types=1.000, volume=1.000, surface_area=1.000, principal_moments=1.000, icp=0.999); surface_types_diff=0.008% (Matched); volume_diff=0.004% (Matched); surface_area_diff=0.001% (Matched); bbox_diff=0.000% (OCCT OBB; gate passed); principal_moments_diff=0.006% (Matched); icp[reference.FCStd vs answer.FCStd: reward=0.999 rmse=2.0991e-04 max_residual=4.8274e-04 mm]","cad_spec_consistency_reason":"spec.json vs answer.FCStd: spec_score=1.000 (30/30 consistent, 0 inconsistent, 0 not_found; failure_budget=10)"},"exception_type":null,"cost_usd":0.6845884,"input_tokens":267085,"output_tokens":19159,"started_at":"2026-10-10T14:12:04.306711Z","finished_at":"2026-10-10T14:16:53.870930Z","candidate":{"path":"jobs/mcp-3-sol-v1--claude-code-opus--freecad-mcp--freecad-f32bdb2966/freecad-f32bdb2966__febDF8f/artifacts/agent-workspace/answer.FCStd","sha256":"f3868c13f9295d558841f7d02dd2433f842dc534b4cb1325dc341ebba0c74da5"},"trial_dir":"jobs/mcp-3-sol-v1--claude-code-opus--freecad-mcp--freecad-f32bdb2966/freecad-f32bdb2966__febDF8f","recovery":null,"job_name":"mcp-3-sol-v1--claude-code-opus--freecad-mcp--freecad-f32bdb2966","result_sha256":"51238c85d0d13dbd49fccb582397143af1e4ad9f7721fd366fa46023768510ea"},{"task":"freecad-6b54209a13","cell":"claude-code-opus--freecad-mcp","model":"anthropic/claude-opus-5-5","agent":"claude-code","reasoning_effort":"high","effort_launch_confirmed":true,"effort_evidence":{"source":"native launch log","option":"--effort high"},"observed_model":"claude-opus-5-5","observed_agent_version":"2.1.289","status":"partial","reward":0.615385,"geometry":0.5000010523547074,"spec":0.8,"details":{"reward":0.615385,"geometry_similarity":0.5000010523547074,"cad_spec_consistency":0.8,"combined":0.6153854124336894,"geometry_similarity_reason":"reference.FCStd vs answer.FCStd: overall=0.500 [solid_count=1 (matched)] (surface_types=1.000, volume=1.000, surface_area=1.000, principal_moments=1.000, icp=0.000); surface_types_diff=0.047% (Matched); volume_diff=0.023% (Matched); surface_area_diff=0.045% (Matched); bbox_diff=0.000% (OCCT OBB; gate passed); principal_moments_diff=0.032% (Matched); icp[reference.FCStd vs answer.FCStd: reward=0.000 rmse=1.7303e+00 max_residual=1.2406e+01 mm]","cad_spec_consistency_reason":"spec.json vs answer.FCStd: spec_score=0.800 (22/24 consistent, 0 inconsistent, 2 not_found; failure_budget=10)"},"exception_type":null,"cost_usd":1.5597778,"input_tokens":909982,"output_tokens":41216,"started_at":"2026-10-10T14:12:06.550060Z","finished_at":"2026-10-10T14:22:12.040944Z","candidate":{"path":"jobs/mcp-3-sol-v1--claude-code-opus--freecad-mcp--freecad-6b54209a13/freecad-6b54209a13__WyBXqFf/artifacts/agent-workspace/answer.FCStd","sha256":"caaa1dab1368004316f476a7218e6f4640fd0d38738721af574e63b6db2d997a"},"trial_dir":"jobs/mcp-3-sol-v1--claude-code-opus--freecad-mcp--freecad-6b54209a13/freecad-6b54209a13__WyBXqFf","recovery":null,"job_name":"mcp-3-sol-v1--claude-code-opus--freecad-mcp--freecad-6b54209a13","result_sha256":"4419f964156eae38d1b6e4f9c372848dfa52d9cda99d93e711d4962beedd941e"},{"task":"freecad-f4e56c467a","cell":"claude-code-opus--freecad-mcp","model":"anthropic/claude-opus-5-5","agent":"claude-code","reasoning_effort":"high","effort_launch_confirmed":true,"effort_evidence":{"source":"native launch log","option":"--effort high"},"observed_model":"claude-opus-5-5","observed_agent_version":"2.1.289","status":"partial","reward":0.663475,"geometry":0.4964181586864047,"spec":1.0,"details":{"reward":0.663475,"geometry_similarity":0.4964181586864047,"cad_spec_consistency":1.0,"combined":0.6634751868049685,"geometry_similarity_reason":"reference.FCStd vs answer.FCStd: overall=0.496 [solid_count=1 (matched)] (surface_types=1.000, volume=0.979, surface_area=1.000, principal_moments=1.000, icp=0.000); surface_types_diff=0.895% (Matched); volume_diff=0.105% (Close); surface_area_diff=0.074% (Matched); bbox_diff=0.101% (OCCT OBB; gate passed); principal_moments_diff=0.060% (Matched); icp[reference.FCStd vs answer.FCStd: reward=0.000 rmse=2.3833e+00 max_residual=1.0199e+01 mm]","cad_spec_consistency_reason":"spec.json vs answer.FCStd: spec_score=1.000 (32/32 consistent, 0 inconsistent, 0 not_found; failure_budget=10)"},"exception_type":null,"cost_usd":0.3287302,"input_tokens":183123,"output_tokens":8629,"started_at":"2026-10-10T14:12:05.691542Z","finished_at":"2026-10-10T14:15:21.569552Z","candidate":{"path":"jobs/mcp-3-sol-v1--claude-code-opus--freecad-mcp--freecad-f4e56c467a/freecad-f4e56c467a__JubAw2a/artifacts/agent-workspace/answer.FCStd","sha256":"d98332ae301e26009dbb9ad5dc80af5cab5aba871579389ba419d85c03f18c2e"},"trial_dir":"jobs/mcp-3-sol-v1--claude-code-opus--freecad-mcp--freecad-f4e56c467a/freecad-f4e56c467a__JubAw2a","recovery":null,"job_name":"mcp-3-sol-v1--claude-code-opus--freecad-mcp--freecad-f4e56c467a","result_sha256":"087eb33cc06acc946a0597fa44203ce58016cea671e4cae4517207949c491bc0"},{"task":"freecad-f32bdb2966","cell":"codex-sol-6-1--freecad-mcp","model":"openai/gpt-6.1-sol","agent":"codex","reasoning_effort":"high","effort_launch_confirmed":true,"effort_evidence":{"source":"native launch log","option":"model_reasoning_effort=high"},"observed_model":"gpt-6.1-sol","observed_agent_version":"0.162.1","status":"partial","reward":0.473345,"geometry":0.3100540850690192,"spec":1.0,"details":{"reward":0.473345,"geometry_similarity":0.3100540850690192,"cad_spec_consistency":1.0,"combined":0.4733454726835713,"geometry_similarity_reason":"reference.FCStd vs answer.FCStd: overall=0.310 [solid_count=1 (matched)] (surface_types=0.960, volume=0.000, surface_area=1.000, principal_moments=0.870, icp=0.000); surface_types_diff=1.095% (Close); volume_diff=1.046% (Far); surface_area_diff=0.841% (Matched); bbox_diff=0.000% (OCCT OBB; gate passed); principal_moments_diff=1.348% (Close); icp[reference.FCStd vs answer.FCStd: reward=0.000 rmse=3.0414e+00 max_residual=1.2694e+01 mm]","cad_spec_consistency_reason":"spec.json vs answer.FCStd: spec_score=1.000 (30/30 consistent, 0 inconsistent, 0 not_found; failure_budget=10)"},"exception_type":null,"cost_usd":0.29229599999999994,"input_tokens":464926,"output_tokens":11758,"started_at":"2026-10-10T13:54:08.737481Z","finished_at":"2026-10-10T14:03:37.034422Z","candidate":{"path":"jobs/mcp-3-sol-v1--codex-sol-6-1--freecad-mcp--freecad-f32bdb2966/freecad-f32bdb2966__fvcjA6h/artifacts/agent-workspace/answer.FCStd","sha256":"270c0b02d8a585fc4c41263457d817d874a79165c18766737a65cc9c5e321383"},"trial_dir":"jobs/mcp-3-sol-v1--codex-sol-6-1--freecad-mcp--freecad-f32bdb2966/freecad-f32bdb2966__fvcjA6h","recovery":null,"job_name":"mcp-3-sol-v1--codex-sol-6-1--freecad-mcp--freecad-f32bdb2966","result_sha256":"06fa2bc0a305432a87a3601b9b8de3455895069fcfbd0d2b2417d6b207dfe048"},{"task":"freecad-6b54209a13","cell":"codex-sol-6-1--freecad-mcp","model":"openai/gpt-6.1-sol","agent":"codex","reasoning_effort":"high","effort_launch_confirmed":true,"effort_evidence":{"source":"native launch log","option":"model_reasoning_effort=high"},"observed_model":"gpt-6.1-sol","observed_agent_version":"0.162.1","status":"partial","reward":0.333304,"geometry":0.21872469861247948,"spec":0.7,"details":{"reward":0.333304,"geometry_similarity":0.21872469861247948,"cad_spec_consistency":0.7,"combined":0.3333039576708206,"geometry_similarity_reason":"reference.FCStd vs answer.FCStd: overall=0.219 [solid_count=1 (matched)] (surface_types=0.000, volume=0.000, surface_area=1.000, principal_moments=0.437, icp=0.000); surface_types_diff=100.000% (Far); volume_diff=2.479% (Far); surface_area_diff=0.484% (Matched); bbox_diff=0.000% (OCCT OBB; gate passed); principal_moments_diff=3.654% (Close); icp[reference.FCStd vs answer.FCStd: reward=0.000 rmse=2.2983e+00 max_residual=1.7622e+01 mm]","cad_spec_consistency_reason":"spec.json vs answer.FCStd: spec_score=0.700 (21/24 consistent, 0 inconsistent, 3 not_found; failure_budget=10)"},"exception_type":null,"cost_usd":0.4785739999999999,"input_tokens":1253819,"output_tokens":21932,"started_at":"2026-10-10T13:55:06.550817Z","finished_at":"2026-10-10T14:12:08.735972Z","candidate":{"path":"jobs/mcp-3-sol-v1--codex-sol-6-1--freecad-mcp--freecad-6b54209a13/freecad-6b54209a13__yo8Vujv/artifacts/agent-workspace/answer.FCStd","sha256":"54488bf92e5971a62a18afb331e95749755f9cb91c1d3b4bdf14c1be47858ad0"},"trial_dir":"jobs/mcp-3-sol-v1--codex-sol-6-1--freecad-mcp--freecad-6b54209a13/freecad-6b54209a13__yo8Vujv","recovery":null,"job_name":"mcp-3-sol-v1--codex-sol-6-1--freecad-mcp--freecad-6b54209a13","result_sha256":"1e10721faa2063fbb45e581efb1996c67b2604aaaa7d632111e58817ef53d935"},{"task":"freecad-f4e56c467a","cell":"codex-sol-6-1--freecad-mcp","model":"openai/gpt-6.1-sol","agent":"codex","reasoning_effort":"high","effort_launch_confirmed":true,"effort_evidence":{"source":"native launch log","option":"model_reasoning_effort=high"},"observed_model":"gpt-6.1-sol","observed_agent_version":"0.162.1","status":"partial","reward":0.710356,"geometry":0.5508153389744929,"spec":1.0,"details":{"reward":0.710356,"geometry_similarity":0.5508153389744929,"cad_spec_consistency":1.0,"combined":0.7103558046294929,"geometry_similarity_reason":"reference.FCStd vs answer.FCStd: overall=0.551 [solid_count=1 (matched)] (surface_types=0.606, volume=1.000, surface_area=1.000, principal_moments=1.000, icp=0.147); surface_types_diff=2.476% (Close); volume_diff=0.061% (Matched); surface_area_diff=0.043% (Matched); bbox_diff=0.057% (OCCT OBB; gate passed); principal_moments_diff=0.023% (Matched); icp[reference.FCStd vs answer.FCStd: reward=0.147 rmse=6.6994e-01 max_residual=1.8211e+00 mm]","cad_spec_consistency_reason":"spec.json vs answer.FCStd: spec_score=1.000 (32/32 consistent, 0 inconsistent, 0 not_found; failure_budget=10)"},"exception_type":null,"cost_usd":0.28946759999999994,"input_tokens":593102,"output_tokens":14343,"started_at":"2026-10-10T13:55:07.045309Z","finished_at":"2026-10-10T14:06:01.327659Z","candidate":{"path":"jobs/mcp-3-sol-v1--codex-sol-6-1--freecad-mcp--freecad-f4e56c467a/freecad-f4e56c467a__evwuW5t/artifacts/agent-workspace/answer.FCStd","sha256":"cd5b8479547c13ce80593d32ecc8c1bbb1c35622182fc779ef5a898336d37367"},"trial_dir":"jobs/mcp-3-sol-v1--codex-sol-6-1--freecad-mcp--freecad-f4e56c467a/freecad-f4e56c467a__evwuW5t","recovery":null,"job_name":"mcp-3-sol-v1--codex-sol-6-1--freecad-mcp--freecad-f4e56c467a","result_sha256":"16e79857973c03437c37172d06248168b1a1c01740fc4bb9db104bd523a1ac91"}]}