{"entries":[{"rank":1,"submission_id":"excel_academic_all_submission_01","result_id":"excel_academic_all_result_01","job_id":"excel_academic_all_job_01","user_id":"excel_academic_public","target_version_label":"InvMem","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":45.0572034208447,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":45.0572034208447,"llm_score":45.0572034208447},"score":45.0572034208447,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":56.775277234181296},"score":56.775277234181296,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":45.9556682147729},"score":45.9556682147729,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":21.7744271232643},"score":21.7744271232643,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":32.6222675791641},"score":32.6222675791641,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":53.1392694063927},"score":53.1392694063927,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":30.770156438026504},"score":30.770156438026504,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":30.322580645161302},"score":30.322580645161302,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":57.5545203588682},"score":57.5545203588682,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":50.485436893203904},"score":50.485436893203904,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":49.2175572519084},"score":49.2175572519084,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":48.0217068820634},"score":48.0217068820634,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":36.2637362637363},"score":36.2637362637363,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":31.5789473684211},"score":31.5789473684211,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":52.1739130434783},"score":52.1739130434783,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":17.9171585989768},"score":17.9171585989768,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":34.375223773719995},"score":34.375223773719995,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":55.5555555555556},"score":55.5555555555556,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":44.2118226600985},"score":44.2118226600985,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":16.6666666666667},"score":16.6666666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":10.526315789473701},"score":10.526315789473701,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":35.393247955747896},"score":35.393247955747896,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":50.2792672028597},"score":50.2792672028597,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":51.123595505618},"score":51.123595505618,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":60.8695652173913},"score":60.8695652173913,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":33.5555555555556},"score":33.5555555555556,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":24.802110817941898},"score":24.802110817941898,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":37.2602739726027},"score":37.2602739726027,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":12},"score":12,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":36.8217054263566},"score":36.8217054263566,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":33.734939759036095},"score":33.734939759036095,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":26.3888888888889},"score":26.3888888888889,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank01","cate":"Submitted (repo)","github_url":"https://github.com/wenxiaof345-ctrl/vanilla-rag-memory","system_name":"InvMem","target_version_label":"InvMem","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":2,"submission_id":"excel_academic_all_submission_02","result_id":"excel_academic_all_result_02","job_id":"excel_academic_all_job_02","user_id":"excel_academic_public","target_version_label":"Refind","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":44.974070824491704,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":44.974070824491704,"llm_score":44.974070824491704},"score":44.974070824491704,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":57.7479071537291},"score":57.7479071537291,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":45.300733872433405},"score":45.300733872433405,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":19.6176562455632},"score":19.6176562455632,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":33.3263049211325},"score":33.3263049211325,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":52.7723418134377},"score":52.7723418134377,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.2972322503008},"score":28.2972322503008,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":30.322580645161302},"score":30.322580645161302,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":58.72846215781},"score":58.72846215781,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":44.6601941747573},"score":44.6601941747573,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":54.1952926208651},"score":54.1952926208651,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":47.2889365397854},"score":47.2889365397854,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":30.7692307692308},"score":30.7692307692308,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":31.5789473684211},"score":31.5789473684211,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":60.8695652173913},"score":60.8695652173913,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":16.046012068739298},"score":16.046012068739298,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":24.1824800095477},"score":24.1824800095477,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":55.5555555555556},"score":55.5555555555556,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":45.7512315270936},"score":45.7512315270936,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.2083333333333},"score":20.2083333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":7.89473684210526},"score":7.89473684210526,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":39.6086159211159},"score":39.6086159211159,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":49.7765862377123},"score":49.7765862377123,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":51.685393258427005},"score":51.685393258427005,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":60.8695652173913},"score":60.8695652173913,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":32.4444444444444},"score":32.4444444444444,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.899736147757302},"score":21.899736147757302,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.1506849315069},"score":33.1506849315069,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":7.000000000000001},"score":7.000000000000001,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":31.5891472868217},"score":31.5891472868217,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":27.710843373493997},"score":27.710843373493997,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank02","cate":"Submitted (repo)","github_url":"https://github.com/imlrz/ReFind","system_name":"Refind","target_version_label":"Refind","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":3,"submission_id":"excel_academic_all_submission_03","result_id":"excel_academic_all_result_03","job_id":"excel_academic_all_job_03","user_id":"excel_academic_public","target_version_label":"ActiveMemoryIndex","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":44.8410926975032,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":44.8410926975032,"llm_score":44.8410926975032},"score":44.8410926975032,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":56.8720373994347},"score":56.8720373994347,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":45.038437157177704},"score":45.038437157177704,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":18.3514353863191},"score":18.3514353863191,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":37.950225062294},"score":37.950225062294,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":54.5662100456621},"score":54.5662100456621,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":29.663056558363397},"score":29.663056558363397,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":32.9032258064516},"score":32.9032258064516,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":57.9467448815275},"score":57.9467448815275,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":44.6601941747573},"score":44.6601941747573,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":51.7620865139949},"score":51.7620865139949,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":47.3350256138881},"score":47.3350256138881,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":26.3736263736264},"score":26.3736263736264,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":23.6842105263158},"score":23.6842105263158,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":15.410025580480099},"score":15.410025580480099,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":26.1913712853563},"score":26.1913712853563,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":48.1481481481482},"score":48.1481481481482,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":54.4950738916256},"score":54.4950738916256,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":24.7916666666667},"score":24.7916666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":5.26315789473684},"score":5.26315789473684,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":44.0789141414141},"score":44.0789141414141,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":54.1108132260947},"score":54.1108132260947,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":58.42696629213479},"score":58.42696629213479,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":55.7971014492754},"score":55.7971014492754,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":32.4444444444444},"score":32.4444444444444,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":25.5936675461741},"score":25.5936675461741,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.972602739726},"score":33.972602739726,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":8},"score":8,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":36.201550387596896},"score":36.201550387596896,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":37.3493975903614},"score":37.3493975903614,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":27.7777777777778},"score":27.7777777777778,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank03","cate":"Submitted (API)","github_url":"https://github.com/linxuhao/ActiveMemoryIndex.git","system_name":"ActiveMemoryIndex","target_version_label":"ActiveMemoryIndex","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":4,"submission_id":"excel_academic_all_submission_04","result_id":"excel_academic_all_result_04","job_id":"excel_academic_all_job_04","user_id":"excel_academic_public","target_version_label":"Hybrid Search v2.0","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":44.572662270698096,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":44.572662270698096,"llm_score":44.572662270698096},"score":44.572662270698096,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":57.8735051098065},"score":57.8735051098065,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":46.8090563120912},"score":46.8090563120912,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":20.9295936621518},"score":20.9295936621518,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":32.0845055327814},"score":32.0845055327814,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":52.5603392041748},"score":52.5603392041748,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":26.744885679903703},"score":26.744885679903703,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":30.322580645161302},"score":30.322580645161302,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":59.1511962272832},"score":59.1511962272832,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":39.8058252427185},"score":39.8058252427185,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":46.8193384223918},"score":46.8193384223918,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":48.976516315226},"score":48.976516315226,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":31.8681318681319},"score":31.8681318681319,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":28.947368421052598},"score":28.947368421052598,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":63.768115942029},"score":63.768115942029,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":18.181080283352998},"score":18.181080283352998,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":28.5687432867884},"score":28.5687432867884,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":48.1481481481482},"score":48.1481481481482,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":43.4729064039409},"score":43.4729064039409,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.8333333333333},"score":20.8333333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":6.14035087719298},"score":6.14035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":42.8476430976431},"score":42.8476430976431,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":49.218051831992796},"score":49.218051831992796,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":50},"score":50,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":61.5942028985507},"score":61.5942028985507,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30.2222222222222},"score":30.2222222222222,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":18.9973614775726},"score":18.9973614775726,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.4246575342466},"score":33.4246575342466,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":34.1472868217054},"score":34.1472868217054,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":28.915662650602396},"score":28.915662650602396,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":31.944444444444397},"score":31.944444444444397,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank04","cate":"Submitted (repo)","github_url":"https://github.com/cydd-1972/hybrid_search","system_name":"Hybrid Search v2.0","target_version_label":"Hybrid Search v2.0","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":5,"submission_id":"excel_academic_all_submission_05","result_id":"excel_academic_all_result_05","job_id":"excel_academic_all_job_05","user_id":"excel_academic_public","target_version_label":"ChronoHybridMem","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":44.333794997242,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":44.333794997242,"llm_score":44.333794997242},"score":44.333794997242,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":55.717357034137905},"score":55.717357034137905,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":44.441915359973},"score":44.441915359973,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":23.299045915325},"score":23.299045915325,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":32.9441918235022},"score":32.9441918235022,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":57.1265492498369},"score":57.1265492498369,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.309265944645002},"score":28.309265944645002,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":35},"score":35,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":56.9666724177594},"score":56.9666724177594,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":42.7184466019418},"score":42.7184466019418,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":47.4204834605598},"score":47.4204834605598,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":47.1769477457083},"score":47.1769477457083,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":24.1758241758242},"score":24.1758241758242,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":18.4210526315789},"score":18.4210526315789,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":50.7246376811594},"score":50.7246376811594,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":20.2978650137741},"score":20.2978650137741,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":22.1741854636591},"score":22.1741854636591,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":55.5555555555556},"score":55.5555555555556,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":46.0591133004926},"score":46.0591133004926,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":22.0833333333333},"score":22.0833333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":4.3859649122807},"score":4.3859649122807,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":42.832671957672005},"score":42.832671957672005,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":53.3288650580876},"score":53.3288650580876,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":57.86516853932579},"score":57.86516853932579,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":67.3913043478261},"score":67.3913043478261,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":31.555555555555497},"score":31.555555555555497,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":24.0105540897098},"score":24.0105540897098,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":32.6027397260274},"score":32.6027397260274,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":3},"score":3,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":36.3178294573643},"score":36.3178294573643,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":30.421686746988},"score":30.421686746988,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":40.2777777777778},"score":40.2777777777778,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank05","cate":"Submitted (repo)","github_url":"https://github.com/Tin11Mn/chrono-hybrid-mem","system_name":"ChronoHybridMem","target_version_label":"ChronoHybridMem","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":6,"submission_id":"excel_academic_all_submission_06","result_id":"excel_academic_all_result_06","job_id":"excel_academic_all_job_06","user_id":"excel_academic_public","target_version_label":"Hybrid Episodic Memory","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":44.2765028098505,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":44.2765028098505,"llm_score":44.2765028098505},"score":44.2765028098505,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":55.505544683626894},"score":55.505544683626894,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":45.9005675659697},"score":45.9005675659697,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":21.563023539767702},"score":21.563023539767702,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":34.2320897493311},"score":34.2320897493311,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":57.289628180039095},"score":57.289628180039095,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.0806257521059},"score":28.0806257521059,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":27.096774193548402},"score":27.096774193548402,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":56.3077984817115},"score":56.3077984817115,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":45.6310679611651},"score":45.6310679611651,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":52.4936386768448},"score":52.4936386768448,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":47.2809406213481},"score":47.2809406213481,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":35.1648351648352},"score":35.1648351648352,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":36.8421052631579},"score":36.8421052631579,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":57.9710144927536},"score":57.9710144927536,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":19.119277187458998},"score":19.119277187458998,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":29.3513545769185},"score":29.3513545769185,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":48.1481481481482},"score":48.1481481481482,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":47.229064039408904},"score":47.229064039408904,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":17.7083333333333},"score":17.7083333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":7.01754385964912},"score":7.01754385964912,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":41.9666305916306},"score":41.9666305916306,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":55.4289544235925},"score":55.4289544235925,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":58.42696629213479},"score":58.42696629213479,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":62.3188405797101},"score":62.3188405797101,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":32.2222222222222},"score":32.2222222222222,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.899736147757302},"score":21.899736147757302,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":32.054794520548},"score":32.054794520548,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":9},"score":9,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":35.3100775193798},"score":35.3100775193798,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":21.6867469879518},"score":21.6867469879518,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank06","cate":"Submitted (repo)","github_url":"https://github.com/tlysanhuo/agent-memory-challenge","system_name":"Hybrid Episodic Memory","target_version_label":"Hybrid Episodic Memory","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":7,"submission_id":"excel_academic_all_submission_07","result_id":"excel_academic_all_result_07","job_id":"excel_academic_all_job_07","user_id":"excel_academic_public","target_version_label":"Chronicle","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":44.1930891194327,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":44.1930891194327,"llm_score":44.1930891194327},"score":44.1930891194327,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":56.203141987388605},"score":56.203141987388605,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":43.9337418187949},"score":43.9337418187949,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":21.2270352386631},"score":21.2270352386631,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":30.404389671630998},"score":30.404389671630998,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":52.57664709719499},"score":52.57664709719499,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.561973525872396},"score":28.561973525872396,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":30.322580645161302},"score":30.322580645161302,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":57.2390729238555},"score":57.2390729238555,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":40.7766990291262},"score":40.7766990291262,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":44.7773536895674},"score":44.7773536895674,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":45.4199250570218},"score":45.4199250570218,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":31.8681318681319},"score":31.8681318681319,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":34.2105263157895},"score":34.2105263157895,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":18.3180014430014},"score":18.3180014430014,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":20.420396228666903},"score":20.420396228666903,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":50},"score":50,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":41.564039408866996},"score":41.564039408866996,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":19.375},"score":19.375,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":5.26315789473684},"score":5.26315789473684,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":40.4159151034151},"score":40.4159151034151,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":49.7765862377122},"score":49.7765862377122,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":52.8089887640449},"score":52.8089887640449,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":60.1449275362319},"score":60.1449275362319,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":32.6666666666667},"score":32.6666666666667,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.899736147757302},"score":21.899736147757302,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.6986301369863},"score":33.6986301369863,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":8},"score":8,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":36.5503875968992},"score":36.5503875968992,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":30.1204819277109},"score":30.1204819277109,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":30.555555555555603},"score":30.555555555555603,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank07","cate":"Submitted (repo)","github_url":"https://github.com/simple-boy/Chronicle-Memory","system_name":"Chronicle","target_version_label":"Chronicle","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":8,"submission_id":"excel_academic_all_submission_08","result_id":"excel_academic_all_result_08","job_id":"excel_academic_all_job_08","user_id":"excel_academic_public","target_version_label":"FlowGrid_AML_Retriever","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":43.9846746114283,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":43.9846746114283,"llm_score":43.9846746114283},"score":43.9846746114283,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":55.593226788432304},"score":55.593226788432304,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":45.1896895466698},"score":45.1896895466698,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":21.128787233438402},"score":21.128787233438402,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":27.8625254056289},"score":27.8625254056289,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":51.2883235485975},"score":51.2883235485975,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":29.235860409145598},"score":29.235860409145598,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":30.322580645161302},"score":30.322580645161302,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":56.352254428341396},"score":56.352254428341396,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":48.5436893203884},"score":48.5436893203884,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":45.629770992366396},"score":45.629770992366396,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":46.485577947802},"score":46.485577947802,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":32.967032967033},"score":32.967032967033,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":36.8421052631579},"score":36.8421052631579,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":57.9710144927536},"score":57.9710144927536,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":17.791305916305898},"score":17.791305916305898,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":24.142797469865098},"score":24.142797469865098,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":53.703703703703695},"score":53.703703703703695,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":37.5},"score":37.5,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.625},"score":20.625,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":5.26315789473684},"score":5.26315789473684,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":38.194534632034596},"score":38.194534632034596,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":47.4754244861483},"score":47.4754244861483,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":50.561797752809},"score":50.561797752809,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":61.5942028985507},"score":61.5942028985507,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":33.1111111111111},"score":33.1111111111111,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.635883905013202},"score":21.635883905013202,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":34.2465753424658},"score":34.2465753424658,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":38.5658914728682},"score":38.5658914728682,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":27.710843373493997},"score":27.710843373493997,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank08","cate":"Submitted (repo)","github_url":"https://github.com/dlxeva/flowgrid-aml-retriever","system_name":"FlowGrid_AML_Retriever","target_version_label":"FlowGrid_AML_Retriever","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":9,"submission_id":"excel_academic_all_submission_09","result_id":"excel_academic_all_result_09","job_id":"excel_academic_all_job_09","user_id":"excel_academic_public","target_version_label":"LLLMemoryAgent","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":43.0617583126756,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":43.0617583126756,"llm_score":43.0617583126756},"score":43.0617583126756,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":54.517666884105196},"score":54.517666884105196,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":42.7019161920528},"score":42.7019161920528,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":24.623548286339002},"score":24.623548286339002,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":30.3188478188478},"score":30.3188478188478,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":53.5061969993477},"score":53.5061969993477,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.6293622141997},"score":27.6293622141997,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":29.6774193548387},"score":29.6774193548387,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":55.5525074764205},"score":55.5525074764205,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":46.6019417475728},"score":46.6019417475728,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":47.64631043257},"score":47.64631043257,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":45.3999367921269},"score":45.3999367921269,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":24.1758241758242},"score":24.1758241758242,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":18.4210526315789},"score":18.4210526315789,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":21.2077758100485},"score":21.2077758100485,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":25.485439789951098},"score":25.485439789951098,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":59.2592592592593},"score":59.2592592592593,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":38.1773399014778},"score":38.1773399014778,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":14.5833333333333},"score":14.5833333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":11.4035087719298},"score":11.4035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":39.670093795093806},"score":39.670093795093806,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":51.3181411974978},"score":51.3181411974978,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":51.685393258427005},"score":51.685393258427005,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":59.4202898550725},"score":59.4202898550725,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30.8888888888889},"score":30.8888888888889,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":23.4828496042216},"score":23.4828496042216,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":30.958904109589003},"score":30.958904109589003,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":5},"score":5,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":34.2635658914729},"score":34.2635658914729,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":27.710843373493997},"score":27.710843373493997,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":31.944444444444397},"score":31.944444444444397,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank09","cate":"Submitted (repo)","github_url":"https://github.com/llLAlisa/memory-agent-submission","system_name":"LLLMemoryAgent","target_version_label":"LLLMemoryAgent","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":10,"submission_id":"excel_academic_all_submission_10","result_id":"excel_academic_all_result_10","job_id":"excel_academic_all_job_10","user_id":"excel_academic_public","target_version_label":"aml-memory-mvp","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":42.952773627552396,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":42.952773627552396,"llm_score":42.952773627552396},"score":42.952773627552396,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":54.830044574907596},"score":54.830044574907596,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":43.2996450523006},"score":43.2996450523006,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":17.9999716046228},"score":17.9999716046228,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":32.311012742047204},"score":32.311012742047204,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":54.3623613829093},"score":54.3623613829093,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.2593261131167},"score":27.2593261131167,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":29.032258064516096},"score":29.032258064516096,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":55.9313031976076},"score":55.9313031976076,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":47.5728155339806},"score":47.5728155339806,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":49.1062340966921},"score":49.1062340966921,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":45.8989237512158},"score":45.8989237512158,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":25.274725274725302},"score":25.274725274725302,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":18.4210526315789},"score":18.4210526315789,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":53.623188405797094},"score":53.623188405797094,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":15.6239341466614},"score":15.6239341466614,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":21.1427377968731},"score":21.1427377968731,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":46.296296296296305},"score":46.296296296296305,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":45.9975369458128},"score":45.9975369458128,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":18.5416666666667},"score":18.5416666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":3.50877192982456},"score":3.50877192982456,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":40.7291967291967},"score":40.7291967291967,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":51.150580875781905},"score":51.150580875781905,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":54.494382022471896},"score":54.494382022471896,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":63.0434782608696},"score":63.0434782608696,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":31.1111111111111},"score":31.1111111111111,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":20.5804749340369},"score":20.5804749340369,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.1506849315069},"score":33.1506849315069,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":2},"score":2,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":34.2054263565891},"score":34.2054263565891,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":30.1204819277109},"score":30.1204819277109,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":27.7777777777778},"score":27.7777777777778,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank10","cate":"Submitted (repo)","github_url":"https://github.com/0xboyu/aml-memory-mvp","system_name":"aml-memory-mvp","target_version_label":"aml-memory-mvp","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":11,"submission_id":"excel_academic_all_submission_11","result_id":"excel_academic_all_result_11","job_id":"excel_academic_all_job_11","user_id":"excel_academic_public","target_version_label":"TraceMem","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":42.5305341347862,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":42.5305341347862,"llm_score":42.5305341347862},"score":42.5305341347862,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":52.45170145683839},"score":52.45170145683839,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":42.5191734163661},"score":42.5191734163661,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":18.3498736405713},"score":18.3498736405713,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":36.340828711518405},"score":36.340828711518405,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":55.471298108284394},"score":55.471298108284394,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.6113116726835},"score":27.6113116726835,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":34.8387096774193},"score":34.8387096774193,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":53.4150851161721},"score":53.4150851161721,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":46.6019417475728},"score":46.6019417475728,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":49.9332061068702},"score":49.9332061068702,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":44.8559172858833},"score":44.8559172858833,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":27.472527472527503},"score":27.472527472527503,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":13.157894736842099},"score":13.157894736842099,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":55.072463768115895},"score":55.072463768115895,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":15.4082218286764},"score":15.4082218286764,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":27.50059672992},"score":27.50059672992,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":50},"score":50,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":47.4753694581281},"score":47.4753694581281,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.2083333333333},"score":20.2083333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":12.280701754386},"score":12.280701754386,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":44.3832070707071},"score":44.3832070707071,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":54.0102770330652},"score":54.0102770330652,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":58.9887640449438},"score":58.9887640449438,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":59.4202898550725},"score":59.4202898550725,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30.666666666666696},"score":30.666666666666696,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.899736147757302},"score":21.899736147757302,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":31.2328767123288},"score":31.2328767123288,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":9},"score":9,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":34.1472868217054},"score":34.1472868217054,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":33.734939759036095},"score":33.734939759036095,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":36.1111111111111},"score":36.1111111111111,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank11","cate":"Submitted (repo)","github_url":"https://github.com/infinigence/Infini-Memory","system_name":"TraceMem","target_version_label":"TraceMem","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":12,"submission_id":"excel_academic_all_submission_12","result_id":"excel_academic_all_result_12","job_id":"excel_academic_all_job_12","user_id":"excel_academic_public","target_version_label":"Memoria","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":42.325474457192605,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":42.325474457192605,"llm_score":42.325474457192605},"score":42.325474457192605,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":52.3408077843009},"score":52.3408077843009,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":42.5308827490163},"score":42.5308827490163,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":18.0348624243973},"score":18.0348624243973,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":33.8931901431901},"score":33.8931901431901,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":56.2133072407045},"score":56.2133072407045,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.460890493381502},"score":27.460890493381502,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":30.322580645161302},"score":30.322580645161302,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":53.92578214860821},"score":53.92578214860821,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":34.9514563106796},"score":34.9514563106796,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":42.5667938931298},"score":42.5667938931298,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":45.3783560808179},"score":45.3783560808179,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":20.8791208791209},"score":20.8791208791209,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":13.157894736842099},"score":13.157894736842099,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":15.044396890987798},"score":15.044396890987798,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":26.740511994271397},"score":26.740511994271397,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":51.851851851851805},"score":51.851851851851805,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":44.088669950738904},"score":44.088669950738904,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":19.1666666666667},"score":19.1666666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":11.4035087719298},"score":11.4035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":42.1288780663781},"score":42.1288780663781,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":54.490616621983904},"score":54.490616621983904,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":55.0561797752809},"score":55.0561797752809,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":60.8695652173913},"score":60.8695652173913,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30},"score":30,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":23.7467018469657},"score":23.7467018469657,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":32.3287671232877},"score":32.3287671232877,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":3},"score":3,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":34.5736434108527},"score":34.5736434108527,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":30.1204819277109},"score":30.1204819277109,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":30.555555555555603},"score":30.555555555555603,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank12","cate":"Submitted (repo)","github_url":"https://github.com/lin-guan/aml-rag-v2","system_name":"Memoria","target_version_label":"Memoria","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":13,"submission_id":"excel_academic_all_submission_13","result_id":"excel_academic_all_result_13","job_id":"excel_academic_all_job_13","user_id":"excel_academic_public","target_version_label":"Raw Memory","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":42.3082085851889,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":42.3082085851889,"llm_score":42.3082085851889},"score":42.3082085851889,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":50.2326049141118},"score":50.2326049141118,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":45.0550404325063},"score":45.0550404325063,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":18.302311383706698},"score":18.302311383706698,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":38.0376663566319},"score":38.0376663566319,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":57.795172863666},"score":57.795172863666,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.1468110709988},"score":28.1468110709988,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":36.1290322580645},"score":36.1290322580645,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":51.260582010582},"score":51.260582010582,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":40.7766990291262},"score":40.7766990291262,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":41.3167938931298},"score":41.3167938931298,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":47.1838228268619},"score":47.1838228268619,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":30.7692307692308},"score":30.7692307692308,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":23.6842105263158},"score":23.6842105263158,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":55.072463768115895},"score":55.072463768115895,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":15.146677817132401},"score":15.146677817132401,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":32.18791025182},"score":32.18791025182,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":48.1481481481482},"score":48.1481481481482,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":45.1354679802956},"score":45.1354679802956,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":21.0416666666667},"score":21.0416666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":20.175438596491198},"score":20.175438596491198,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":47.961670274170295},"score":47.961670274170295,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":54.5129579982127},"score":54.5129579982127,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":57.30337078651681},"score":57.30337078651681,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":66.6666666666667},"score":66.6666666666667,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":32.8888888888889},"score":32.8888888888889,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":22.6912928759894},"score":22.6912928759894,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":32.8767123287671},"score":32.8767123287671,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":5},"score":5,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":32.015503875968996},"score":32.015503875968996,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":38.5542168674699},"score":38.5542168674699,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank13","cate":"Submitted (API)","github_url":"https://github.com/skadn12/RecMe","system_name":"Raw Memory","target_version_label":"Raw Memory","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":14,"submission_id":"excel_academic_all_submission_14","result_id":"excel_academic_all_result_14","job_id":"excel_academic_all_job_14","user_id":"excel_academic_public","target_version_label":"fenghe","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":42.255689156995004,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":42.255689156995004,"llm_score":42.255689156995004},"score":42.255689156995004,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":53.0477821265492},"score":53.0477821265492,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":41.206332236301904},"score":41.206332236301904,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":16.2384643779993},"score":16.2384643779993,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":33.6922990802301},"score":33.6922990802301,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":51.019243313763894},"score":51.019243313763894,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.1708784596871},"score":28.1708784596871,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":31.6129032258064},"score":31.6129032258064,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":54.3355762594893},"score":54.3355762594893,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":41.747572815534},"score":41.747572815534,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":42.7544529262087},"score":42.7544529262087,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":43.8963886990203},"score":43.8963886990203,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":24.1758241758242},"score":24.1758241758242,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":15.7894736842105},"score":15.7894736842105,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":59.4202898550725},"score":59.4202898550725,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":13.382854519218098},"score":13.382854519218098,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":24.453395393245},"score":24.453395393245,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":48.1481481481482},"score":48.1481481481482,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":46.551724137931},"score":46.551724137931,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":17.9166666666667},"score":17.9166666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":6.14035087719298},"score":6.14035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":42.5332792207792},"score":42.5332792207792,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":48.9834673815907},"score":48.9834673815907,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":53.3707865168539},"score":53.3707865168539,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":32},"score":32,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.3720316622691},"score":21.3720316622691,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.6986301369863},"score":33.6986301369863,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":37.751937984496095},"score":37.751937984496095,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":28.915662650602396},"score":28.915662650602396,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":34.7222222222222},"score":34.7222222222222,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank14","cate":"Submitted (repo)","github_url":"https://github.com/to1a/fenghe","system_name":"fenghe","target_version_label":"fenghe","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":15,"submission_id":"excel_academic_all_submission_15","result_id":"excel_academic_all_result_15","job_id":"excel_academic_all_job_15","user_id":"excel_academic_public","target_version_label":"AMC-Memory","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":42.220606515339895,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":42.220606515339895,"llm_score":42.220606515339895},"score":42.220606515339895,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":52.922374429223694},"score":52.922374429223694,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":37.9285812821473},"score":37.9285812821473,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":19.7834852486015},"score":19.7834852486015,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":28.9985014985015},"score":28.9985014985015,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":53.1474233529028},"score":53.1474233529028,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.315282791817097},"score":28.315282791817097,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":32.258064516129},"score":32.258064516129,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":54.2028985507246},"score":54.2028985507246,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":39.8058252427185},"score":39.8058252427185,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":42.4936386768448},"score":42.4936386768448,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":39.549974643353195},"score":39.549974643353195,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":23.0769230769231},"score":23.0769230769231,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":15.7894736842105},"score":15.7894736842105,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":46.3768115942029},"score":46.3768115942029,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":17.683818706546},"score":17.683818706546,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":19.2403628117914},"score":19.2403628117914,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":44.4444444444444},"score":44.4444444444444,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":38.9778325123153},"score":38.9778325123153,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.2083333333333},"score":20.2083333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":6.14035087719298},"score":6.14035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":38.6655844155844},"score":38.6655844155844,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":50.8266309204647},"score":50.8266309204647,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":53.9325842696629},"score":53.9325842696629,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":59.4202898550725},"score":59.4202898550725,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":32.2222222222222},"score":32.2222222222222,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.1081794195251},"score":21.1081794195251,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":32.8767123287671},"score":32.8767123287671,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":7.000000000000001},"score":7.000000000000001,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":35.426356589147304},"score":35.426356589147304,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":36.144578313253},"score":36.144578313253,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":27.7777777777778},"score":27.7777777777778,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank15","cate":"Submitted (repo)","github_url":"https://github.com/cxy935/AMC-Memo","system_name":"AMC-Memory","target_version_label":"AMC-Memory","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":16,"submission_id":"excel_academic_all_submission_16","result_id":"excel_academic_all_result_16","job_id":"excel_academic_all_job_16","user_id":"excel_academic_public","target_version_label":"MemMachine","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":42.175696093244106,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":42.175696093244106,"llm_score":42.175696093244106},"score":42.175696093244106,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":63.4703196347032},"score":63.4703196347032,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":53.1107738998483},"score":53.1107738998483,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":37.9248658318426},"score":37.9248658318426,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":38.7267904509284},"score":38.7267904509284,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":44.2270058708415},"score":44.2270058708415,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":11.6967509025271},"score":11.6967509025271,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":30.322580645161302},"score":30.322580645161302,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":64.4444444444445},"score":64.4444444444445,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":53.398058252427205},"score":53.398058252427205,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":58.015267175572504},"score":58.015267175572504,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":55.3480475382003},"score":55.3480475382003,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":38.4615384615385},"score":38.4615384615385,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":36.8421052631579},"score":36.8421052631579,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":63.768115942029},"score":63.768115942029,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":35.9504132231405},"score":35.9504132231405,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":41.3533834586466},"score":41.3533834586466,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":57.4074074074074},"score":57.4074074074074,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":40.8866995073892},"score":40.8866995073892,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":0},"score":0,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":28.947368421052598},"score":28.947368421052598,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":50},"score":50,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":46.3806970509383},"score":46.3806970509383,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":38.2022471910112},"score":38.2022471910112,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":38.4057971014493},"score":38.4057971014493,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":14.2222222222222},"score":14.2222222222222,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":12.664907651715},"score":12.664907651715,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":1.0958904109589},"score":1.0958904109589,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":1},"score":1,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":26.046511627906998},"score":26.046511627906998,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":42.1686746987952},"score":42.1686746987952,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":16.6666666666667},"score":16.6666666666667,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank16","cate":"Submitted (repo)","github_url":"https://github.com/MemMachine/MemMachine","system_name":"MemMachine","target_version_label":"MemMachine","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":17,"submission_id":"excel_academic_all_submission_17","result_id":"excel_academic_all_result_17","job_id":"excel_academic_all_job_17","user_id":"excel_academic_public","target_version_label":"causal-memory","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":42.1036456887957,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":42.1036456887957,"llm_score":42.1036456887957},"score":42.1036456887957,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":53.1971624266145},"score":53.1971624266145,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":40.9540670086952},"score":40.9540670086952,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":21.107561688957},"score":21.107561688957,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":31.6939286766873},"score":31.6939286766873,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":54.2074363992172},"score":54.2074363992172,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":26.690734055354998},"score":26.690734055354998,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":25.161290322580598},"score":25.161290322580598,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":54.203761214630795},"score":54.203761214630795,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":45.6310679611651},"score":45.6310679611651,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":45.795165394402},"score":45.795165394402,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":42.765246449457},"score":42.765246449457,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":28.571428571428598},"score":28.571428571428598,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":18.4210526315789},"score":18.4210526315789,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":52.1739130434783},"score":52.1739130434783,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":17.3535681490227},"score":17.3535681490227,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":25.933285594939697},"score":25.933285594939697,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":59.2592592592593},"score":59.2592592592593,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":42.4261083743842},"score":42.4261083743842,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.2083333333333},"score":20.2083333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":7.89473684210526},"score":7.89473684210526,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":40.6018518518519},"score":40.6018518518519,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":51.2064343163539},"score":51.2064343163539,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":54.494382022471896},"score":54.494382022471896,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":62.3188405797101},"score":62.3188405797101,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30.8888888888889},"score":30.8888888888889,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.3720316622691},"score":21.3720316622691,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":30.1369863013699},"score":30.1369863013699,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":35.1937984496124},"score":35.1937984496124,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":22.891566265060202},"score":22.891566265060202,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":27.7777777777778},"score":27.7777777777778,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank17","cate":"Submitted (repo)","github_url":"https://github.com/JingxuanC/causal-memory","system_name":"causal-memory","target_version_label":"causal-memory","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":18,"submission_id":"excel_academic_all_submission_18","result_id":"excel_academic_all_result_18","job_id":"excel_academic_all_job_18","user_id":"excel_academic_public","target_version_label":"aml-rag","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":42.0917050309453,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":42.0917050309453,"llm_score":42.0917050309453},"score":42.0917050309453,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":51.95129375951289},"score":51.95129375951289,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":42.4978869553983},"score":42.4978869553983,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":18.0720248743505},"score":18.0720248743505,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":33.802672423362104},"score":33.802672423362104,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":54.4520547945205},"score":54.4520547945205,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.0264741275571},"score":28.0264741275571,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":29.6774193548387},"score":29.6774193548387,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":52.933977455716594},"score":52.933977455716594,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":46.6019417475728},"score":46.6019417475728,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":48.8422391857506},"score":48.8422391857506,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":45.1716595986545},"score":45.1716595986545,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":25.274725274725302},"score":25.274725274725302,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":13.157894736842099},"score":13.157894736842099,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":55.072463768115895},"score":55.072463768115895,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":14.260871704053498},"score":14.260871704053498,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":25.580916577157197},"score":25.580916577157197,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":57.4074074074074},"score":57.4074074074074,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":45.1354679802956},"score":45.1354679802956,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":17.2916666666667},"score":17.2916666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":7.89473684210526},"score":7.89473684210526,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":44.6851250601251},"score":44.6851250601251,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":51.0053619302949},"score":51.0053619302949,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":57.30337078651681},"score":57.30337078651681,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":63.768115942029},"score":63.768115942029,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":32.4444444444444},"score":32.4444444444444,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":20.3166226912929},"score":20.3166226912929,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.4246575342466},"score":33.4246575342466,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":5},"score":5,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":36.8217054263566},"score":36.8217054263566,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":30.1204819277109},"score":30.1204819277109,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":29.166666666666703},"score":29.166666666666703,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank18","cate":"Submitted (repo)","github_url":"https://github.com/lin-guan/aml-rag","system_name":"aml-rag","target_version_label":"aml-rag","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":19,"submission_id":"excel_academic_all_submission_19","result_id":"excel_academic_all_result_19","job_id":"excel_academic_all_job_19","user_id":"excel_academic_public","target_version_label":"Mem0","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":42.067774660047505,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":42.067774660047505,"llm_score":42.067774660047505},"score":42.067774660047505,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":58.721461187214594},"score":58.721461187214594,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":49.013657056145696},"score":49.013657056145696,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":26.833631484794303},"score":26.833631484794303,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":35.2785145888594},"score":35.2785145888594,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":44.2270058708415},"score":44.2270058708415,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":18.1949458483755},"score":18.1949458483755,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":52.9032258064516},"score":52.9032258064516,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":60.1449275362319},"score":60.1449275362319,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":42.7184466019417},"score":42.7184466019417,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":45.0381679389313},"score":45.0381679389313,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":51.2733446519525},"score":51.2733446519525,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":25.274725274725302},"score":25.274725274725302,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":26.315789473684198},"score":26.315789473684198,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":59.4202898550725},"score":59.4202898550725,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":25.8264462809917},"score":25.8264462809917,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":31.5789473684211},"score":31.5789473684211,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":44.4444444444444},"score":44.4444444444444,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":44.3349753694581},"score":44.3349753694581,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":11.6666666666667},"score":11.6666666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":3.50877192982456},"score":3.50877192982456,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":65},"score":65,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":44.2359249329759},"score":44.2359249329759,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":44.9438202247191},"score":44.9438202247191,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":44.2028985507246},"score":44.2028985507246,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":20.4444444444444},"score":20.4444444444444,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":14.775725593667499},"score":14.775725593667499,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":17.8082191780822},"score":17.8082191780822,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":2},"score":2,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":24.1860465116279},"score":24.1860465116279,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":77.1084337349398},"score":77.1084337349398,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":25},"score":25,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank19","cate":"Submitted (repo)","github_url":"https://github.com/mem0ai/mem0","system_name":"Mem0","target_version_label":"Mem0","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":20,"submission_id":"excel_academic_all_submission_20","result_id":"excel_academic_all_result_20","job_id":"excel_academic_all_job_20","user_id":"excel_academic_public","target_version_label":"Mem517","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":41.8929477781733,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":41.8929477781733,"llm_score":41.8929477781733},"score":41.8929477781733,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":51.467873450750204},"score":51.467873450750204,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":41.989015452596604},"score":41.989015452596604,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":16.2732842093307},"score":16.2732842093307,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":31.1988107677763},"score":31.1988107677763,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":51.761252446184},"score":51.761252446184,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.622141997593296},"score":28.622141997593296,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":36.7741935483871},"score":36.7741935483871,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":52.422532781228405},"score":52.422532781228405,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":41.747572815534},"score":41.747572815534,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":47.2773536895674},"score":47.2773536895674,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":44.602311007234604},"score":44.602311007234604,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":23.0769230769231},"score":23.0769230769231,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":13.157894736842099},"score":13.157894736842099,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":66.6666666666667},"score":66.6666666666667,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":12.5966237045782},"score":12.5966237045782,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":26.8553825038787},"score":26.8553825038787,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":53.703703703703695},"score":53.703703703703695,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":42.0566502463054},"score":42.0566502463054,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":17.2916666666667},"score":17.2916666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":11.4035087719298},"score":11.4035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":32.0741943241943},"score":32.0741943241943,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":52.1447721179625},"score":52.1447721179625,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":50},"score":50,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":50.7246376811594},"score":50.7246376811594,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30.444444444444503},"score":30.444444444444503,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":24.0105540897098},"score":24.0105540897098,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.6986301369863},"score":33.6986301369863,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":7.000000000000001},"score":7.000000000000001,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":37.4031007751938},"score":37.4031007751938,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":43.3734939759036},"score":43.3734939759036,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":29.166666666666703},"score":29.166666666666703,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank20","cate":"Submitted (repo)","github_url":"https://github.com/CodeWater517/Mem517","system_name":"Mem517","target_version_label":"Mem517","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":21,"submission_id":"excel_academic_all_submission_21","result_id":"excel_academic_all_result_21","job_id":"excel_academic_all_job_21","user_id":"excel_academic_public","target_version_label":"AM-Link","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":41.8192766055922,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":41.8192766055922,"llm_score":41.8192766055922},"score":41.8192766055922,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":52.8608936725375},"score":52.8608936725375,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":42.8304341923462},"score":42.8304341923462,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":17.6418349092768},"score":17.6418349092768,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":30.848538434745297},"score":30.848538434745297,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":51.3046314416177},"score":51.3046314416177,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.503008423586},"score":27.503008423586,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":29.6774193548387},"score":29.6774193548387,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":53.7996894409938},"score":53.7996894409938,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":36.893203883495204},"score":36.893203883495204,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":50.025445292620894},"score":50.025445292620894,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":44.6948321438984},"score":44.6948321438984,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":29.6703296703297},"score":29.6703296703297,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":31.5789473684211},"score":31.5789473684211,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":57.9710144927536},"score":57.9710144927536,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":14.7970779220779},"score":14.7970779220779,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":24.3367346938775},"score":24.3367346938775,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":48.1481481481482},"score":48.1481481481482,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":43.3497536945813},"score":43.3497536945813,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":18.75},"score":18.75,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":2.63157894736842},"score":2.63157894736842,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":42.1649831649832},"score":42.1649831649832,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":47.765862377122396},"score":47.765862377122396,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":52.8089887640449},"score":52.8089887640449,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":60.8695652173913},"score":60.8695652173913,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30.444444444444503},"score":30.444444444444503,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":22.4274406332454},"score":22.4274406332454,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":32.6027397260274},"score":32.6027397260274,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":5},"score":5,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":35.7751937984496},"score":35.7751937984496,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":30.1204819277109},"score":30.1204819277109,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":29.166666666666703},"score":29.166666666666703,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank21","cate":"Submitted (API)","github_url":"https://github.com/Graygoo7529/AM-L","system_name":"AM-Link","target_version_label":"AM-Link","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":22,"submission_id":"excel_academic_all_submission_22","result_id":"excel_academic_all_result_22","job_id":"excel_academic_all_job_22","user_id":"excel_academic_public","target_version_label":"SQLite-FTS-Baseline","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":41.7928187240728,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":41.7928187240728,"llm_score":41.7928187240728},"score":41.7928187240728,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":51.4748043052838},"score":51.4748043052838,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":38.436721978072505},"score":38.436721978072505,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":21.8784961808218},"score":21.8784961808218,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":33.3984837002078},"score":33.3984837002078,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":55.088062622309195},"score":55.088062622309195,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.501805054151603},"score":28.501805054151603,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":27.741935483871},"score":27.741935483871,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":52.236628709454806},"score":52.236628709454806,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":43.6893203883495},"score":43.6893203883495,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":50.5057251908397},"score":50.5057251908397,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":40.4580641486414},"score":40.4580641486414,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":27.472527472527503},"score":27.472527472527503,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":28.947368421052598},"score":28.947368421052598,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":44.927536231884105},"score":44.927536231884105,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":18.4505772005772},"score":18.4505772005772,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":24.4742809404464},"score":24.4742809404464,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":53.703703703703695},"score":53.703703703703695,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":45.4433497536946},"score":45.4433497536946,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20},"score":20,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":7.01754385964912},"score":7.01754385964912,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":42.7704725829726},"score":42.7704725829726,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":52.1447721179625},"score":52.1447721179625,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":55.6179775280899},"score":55.6179775280899,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":63.0434782608696},"score":63.0434782608696,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":33.1111111111111},"score":33.1111111111111,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.635883905013202},"score":21.635883905013202,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":32.8767123287671},"score":32.8767123287671,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":9},"score":9,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":35.232558139534895},"score":35.232558139534895,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":25.3012048192771},"score":25.3012048192771,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":30.555555555555603},"score":30.555555555555603,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank22","cate":"Submitted (API)","github_url":"https://github.com/FantaSmallhamster/ldbd-sqlite-fts-baseline","system_name":"SQLite-FTS-Baseline","target_version_label":"SQLite-FTS-Baseline","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":23,"submission_id":"excel_academic_all_submission_23","result_id":"excel_academic_all_result_23","job_id":"excel_academic_all_job_23","user_id":"excel_academic_public","target_version_label":"fdmem","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":41.2838371847114,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":41.2838371847114,"llm_score":41.2838371847114},"score":41.2838371847114,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":51.5412046096978},"score":51.5412046096978,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":41.5357591742266},"score":41.5357591742266,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":20.8603799301474},"score":20.8603799301474,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":31.008383379073003},"score":31.008383379073003,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":50.2446183953033},"score":50.2446183953033,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.051744885679902},"score":27.051744885679902,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":32.9032258064516},"score":32.9032258064516,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":52.934897630549806},"score":52.934897630549806,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":34.9514563106796},"score":34.9514563106796,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":39.8346055979644},"score":39.8346055979644,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":42.736952964032795},"score":42.736952964032795,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":28.571428571428598},"score":28.571428571428598,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":31.5789473684211},"score":31.5789473684211,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":52.1739130434783},"score":52.1739130434783,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":19.1341991341991},"score":19.1341991341991,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":19.2552810597923},"score":19.2552810597923,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":40.7407407407407},"score":40.7407407407407,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":41.7487684729064},"score":41.7487684729064,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":22.5},"score":22.5,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":7.01754385964912},"score":7.01754385964912,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":40.2526755651756},"score":40.2526755651756,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":51.407506702412896},"score":51.407506702412896,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":52.8089887640449},"score":52.8089887640449,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":47.1014492753623},"score":47.1014492753623,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":31.1111111111111},"score":31.1111111111111,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":20.3166226912929},"score":20.3166226912929,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.1506849315069},"score":33.1506849315069,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":4},"score":4,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":30.1204819277109},"score":30.1204819277109,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":36.1111111111111},"score":36.1111111111111,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank23","cate":"Submitted (API)","github_url":"https://github.com/MrZLeo/fdmem","system_name":"fdmem","target_version_label":"fdmem","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":24,"submission_id":"excel_academic_all_submission_24","result_id":"excel_academic_all_result_24","job_id":"excel_academic_all_job_24","user_id":"excel_academic_public","target_version_label":"nano-memory_chris","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":41.1810973040198,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":41.1810973040198,"llm_score":41.1810973040198},"score":41.1810973040198,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":50.996548162644096},"score":50.996548162644096,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":39.1627224992172},"score":39.1627224992172,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":19.6169463611324},"score":19.6169463611324,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":30.5315039366764},"score":30.5315039366764,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":54.8923679060665},"score":54.8923679060665,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.7496991576414},"score":27.7496991576414,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":22.5806451612903},"score":22.5806451612903,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":51.9238842880147},"score":51.9238842880147,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":43.6893203883495},"score":43.6893203883495,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":46.4472010178117},"score":46.4472010178117,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":40.7610087045571},"score":40.7610087045571,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":26.3736263736264},"score":26.3736263736264,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":23.6842105263158},"score":23.6842105263158,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":46.3768115942029},"score":46.3768115942029,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":16.6650268922996},"score":16.6650268922996,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":20.984007638143},"score":20.984007638143,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":51.851851851851805},"score":51.851851851851805,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":41.1330049261084},"score":41.1330049261084,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":22.0833333333333},"score":22.0833333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":6.14035087719298},"score":6.14035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":41.0062830687831},"score":41.0062830687831,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":52.680965147453094},"score":52.680965147453094,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":55.0561797752809},"score":55.0561797752809,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":60.8695652173913},"score":60.8695652173913,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30.666666666666696},"score":30.666666666666696,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":22.6912928759894},"score":22.6912928759894,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":32.054794520548},"score":32.054794520548,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":7.000000000000001},"score":7.000000000000001,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":35.0387596899225},"score":35.0387596899225,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":18.0722891566265},"score":18.0722891566265,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":27.7777777777778},"score":27.7777777777778,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank24","cate":"Submitted (repo)","github_url":"https://github.com/himalalps/nano-memory","system_name":"nano-memory_chris","target_version_label":"nano-memory_chris","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":25,"submission_id":"excel_academic_all_submission_25","result_id":"excel_academic_all_result_25","job_id":"excel_academic_all_job_25","user_id":"excel_academic_public","target_version_label":"aml-memory-baseline","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":40.7848578442686,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":40.7848578442686,"llm_score":40.7848578442686},"score":40.7848578442686,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":50.878152859317204},"score":50.878152859317204,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":37.4195455530812},"score":37.4195455530812,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":20.987804185478602},"score":20.987804185478602,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":30.042199180130204},"score":30.042199180130204,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":53.155577299412904},"score":53.155577299412904,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.4729241877256},"score":27.4729241877256,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":25.806451612903196},"score":25.806451612903196,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":52.040171382562704},"score":52.040171382562704,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":42.7184466019418},"score":42.7184466019418,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":44.4815521628499},"score":44.4815521628499,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":39.3200008819703},"score":39.3200008819703,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":26.3736263736264},"score":26.3736263736264,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":15.7894736842105},"score":15.7894736842105,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":43.4782608695652},"score":43.4782608695652,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":18.2483110324019},"score":18.2483110324019,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":22.7983649600191},"score":22.7983649600191,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":48.1481481481482},"score":48.1481481481482,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":41.0098522167488},"score":41.0098522167488,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":22.5},"score":22.5,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":5.26315789473684},"score":5.26315789473684,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":40.0151515151515},"score":40.0151515151515,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":50.0335120643432},"score":50.0335120643432,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":52.8089887640449},"score":52.8089887640449,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":61.5942028985507},"score":61.5942028985507,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30},"score":30,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":24.2744063324538},"score":24.2744063324538,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":30.4109589041096},"score":30.4109589041096,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":7.000000000000001},"score":7.000000000000001,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":35.581395348837205},"score":35.581395348837205,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":24.0963855421687},"score":24.0963855421687,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":27.7777777777778},"score":27.7777777777778,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank25","cate":"Submitted (repo)","github_url":"https://github.com/richrichgo/aml-memory-ba","system_name":"aml-memory-baseline","target_version_label":"aml-memory-baseline","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":26,"submission_id":"excel_academic_all_submission_26","result_id":"excel_academic_all_result_26","job_id":"excel_academic_all_job_26","user_id":"excel_academic_public","target_version_label":"TencentDB Agent Memory","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":40.7796050449299,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":40.7796050449299,"llm_score":40.7796050449299},"score":40.7796050449299,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":51.1051859099804},"score":51.1051859099804,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":36.6693243950452},"score":36.6693243950452,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":23.1264020217509},"score":23.1264020217509,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":32.074874359357096},"score":32.074874359357096,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":53.587736464448795},"score":53.587736464448795,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":26.2214199759326},"score":26.2214199759326,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":27.096774193548402},"score":27.096774193548402,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":52.32867494824019},"score":52.32867494824019,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":38.8349514563107},"score":38.8349514563107,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":43.0470737913486},"score":43.0470737913486,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":38.1410607408061},"score":38.1410607408061,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":27.472527472527503},"score":27.472527472527503,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":21.052631578947402},"score":21.052631578947402,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":37.6811594202899},"score":37.6811594202899,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":20.924914731732898},"score":20.924914731732898,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":26.1478100011935},"score":26.1478100011935,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":48.1481481481482},"score":48.1481481481482,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":42.3645320197044},"score":42.3645320197044,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.4166666666667},"score":20.4166666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":7.01754385964912},"score":7.01754385964912,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":44.8704605579606},"score":44.8704605579606,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":52.2341376228776},"score":52.2341376228776,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":53.9325842696629},"score":53.9325842696629,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":57.2463768115942},"score":57.2463768115942,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":27.5555555555556},"score":27.5555555555556,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":20.052770448548802},"score":20.052770448548802,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.4246575342466},"score":33.4246575342466,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":10},"score":10,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":32.6356589147287},"score":32.6356589147287,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":25.3012048192771},"score":25.3012048192771,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":29.166666666666703},"score":29.166666666666703,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank26","cate":"Evaluated","github_url":"https://github.com/TencentCloud/tencentdb-agent-memory","system_name":"TencentDB Agent Memory","target_version_label":"TencentDB Agent Memory","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":27,"submission_id":"excel_academic_all_submission_27","result_id":"excel_academic_all_result_27","job_id":"excel_academic_all_job_27","user_id":"excel_academic_public","target_version_label":"Hindsight AML Adapter aml-v0.3.0","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":40.7550992487858,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":40.7550992487858,"llm_score":40.7550992487858},"score":40.7550992487858,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":51.010246792781},"score":51.010246792781,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":42.9593627583006},"score":42.9593627583006,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":17.9372178209388},"score":17.9372178209388,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":34.4823949996364},"score":34.4823949996364,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":54.30528375733859},"score":54.30528375733859,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":25.4181708784597},"score":25.4181708784597,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":32.258064516129},"score":32.258064516129,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":51.9866862203819},"score":51.9866862203819,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":39.8058252427185},"score":39.8058252427185,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":46.6762086513995},"score":46.6762086513995,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":45.3484211506283},"score":45.3484211506283,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":25.274725274725302},"score":25.274725274725302,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":26.315789473684198},"score":26.315789473684198,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":14.931621408894099},"score":14.931621408894099,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":25.0143215180809},"score":25.0143215180809,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":48.1481481481482},"score":48.1481481481482,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":48.8300492610837},"score":48.8300492610837,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":23.125},"score":23.125,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":6.14035087719298},"score":6.14035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":39.7893819143819},"score":39.7893819143819,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":53.485254691689},"score":53.485254691689,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":56.1797752808989},"score":56.1797752808989,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":29.1111111111111},"score":29.1111111111111,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":17.9419525065963},"score":17.9419525065963,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":29.315068493150697},"score":29.315068493150697,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":12},"score":12,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":31.6472868217054},"score":31.6472868217054,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":32.530120481927696},"score":32.530120481927696,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":31.944444444444397},"score":31.944444444444397,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank27","cate":"Submitted (repo)","github_url":"https://github.com/Flipped111/hindsight-aml","system_name":"Hindsight AML Adapter aml-v0.3.0","target_version_label":"Hindsight AML Adapter aml-v0.3.0","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":28,"submission_id":"excel_academic_all_submission_28","result_id":"excel_academic_all_result_28","job_id":"excel_academic_all_job_28","user_id":"excel_academic_public","target_version_label":"agent-memory-version1_B1","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":40.6835553587118,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":40.6835553587118,"llm_score":40.6835553587118},"score":40.6835553587118,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":50.1455751250272},"score":50.1455751250272,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":38.557951069332006},"score":38.557951069332006,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":23.6220433313457},"score":23.6220433313457,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":27.8427415496381},"score":27.8427415496381,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":53.2126549249837},"score":53.2126549249837,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.0445246690734},"score":28.0445246690734,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":26.4516129032258},"score":26.4516129032258,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":51.0718886588452},"score":51.0718886588452,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":39.8058252427185},"score":39.8058252427185,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":46.291348600508904},"score":46.291348600508904,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":40.763480058896},"score":40.763480058896,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":26.3736263736264},"score":26.3736263736264,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":18.4210526315789},"score":18.4210526315789,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":49.2753623188406},"score":49.2753623188406,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":20.8775252525253},"score":20.8775252525253,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":25.223475355054298},"score":25.223475355054298,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":51.851851851851805},"score":51.851851851851805,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":36.8226600985222},"score":36.8226600985222,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.8333333333333},"score":20.8333333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":5.26315789473684},"score":5.26315789473684,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":40.3618927368927},"score":40.3618927368927,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":49.575513851653305},"score":49.575513851653305,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":53.9325842696629},"score":53.9325842696629,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":63.0434782608696},"score":63.0434782608696,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":28.888888888888896},"score":28.888888888888896,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":23.2189973614776},"score":23.2189973614776,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":34.794520547945204},"score":34.794520547945204,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":7.000000000000001},"score":7.000000000000001,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":35.5426356589147},"score":35.5426356589147,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":25.3012048192771},"score":25.3012048192771,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":27.7777777777778},"score":27.7777777777778,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank28","cate":"Submitted (repo)","github_url":"https://github.com/hikiko666/agent-memory_version1","system_name":"agent-memory-version1_B1","target_version_label":"agent-memory-version1_B1","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":29,"submission_id":"excel_academic_all_submission_29","result_id":"excel_academic_all_result_29","job_id":"excel_academic_all_job_29","user_id":"excel_academic_public","target_version_label":"SimpleMem","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":40.4856044020732,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":40.4856044020732,"llm_score":40.4856044020732},"score":40.4856044020732,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":50.043705153294205},"score":50.043705153294205,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":43.5456915786961},"score":43.5456915786961,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":15.9091134963228},"score":15.9091134963228,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":33.6142258125017},"score":33.6142258125017,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":51.3454011741683},"score":51.3454011741683,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":25.3128760529483},"score":25.3128760529483,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":41.9354838709677},"score":41.9354838709677,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":50.819185645272604},"score":50.819185645272604,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":39.8058252427185},"score":39.8058252427185,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":45.871501272264595},"score":45.871501272264595,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":45.8346532264189},"score":45.8346532264189,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":25.274725274725302},"score":25.274725274725302,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":21.052631578947402},"score":21.052631578947402,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":55.072463768115895},"score":55.072463768115895,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":12.795856290174502},"score":12.795856290174502,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":23.821010860484602},"score":23.821010860484602,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":48.1481481481482},"score":48.1481481481482,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":42.241379310344804},"score":42.241379310344804,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":15.8333333333333},"score":15.8333333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":12.280701754386},"score":12.280701754386,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":44.959385521885494},"score":44.959385521885494,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":49.4302949061662},"score":49.4302949061662,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":50.561797752809},"score":50.561797752809,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":28.888888888888896},"score":28.888888888888896,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":19.5250659630607},"score":19.5250659630607,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":29.315068493150697},"score":29.315068493150697,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":4},"score":4,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":32.8294573643411},"score":32.8294573643411,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":44.578313253012},"score":44.578313253012,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":38.8888888888889},"score":38.8888888888889,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank29","cate":"Submitted (repo)","github_url":"https://github.com/aiming-lab/SimpleMem","system_name":"SimpleMem","target_version_label":"SimpleMem","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":30,"submission_id":"excel_academic_all_submission_30","result_id":"excel_academic_all_result_30","job_id":"excel_academic_all_job_30","user_id":"excel_academic_public","target_version_label":"memfusion","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":40.2492899454911,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":40.2492899454911,"llm_score":40.2492899454911},"score":40.2492899454911,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":47.1886279626006},"score":47.1886279626006,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":38.0584459378086},"score":38.0584459378086,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":14.7739018087855},"score":14.7739018087855,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":28.200496821186498},"score":28.200496821186498,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":50.4566210045662},"score":50.4566210045662,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":30.8784596871239},"score":30.8784596871239,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":28.3870967741935},"score":28.3870967741935,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":47.557050839659496},"score":47.557050839659496,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":49.5145631067961},"score":49.5145631067961,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":45.1208651399491},"score":45.1208651399491,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":38.6765974074972},"score":38.6765974074972,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":37.3626373626374},"score":37.3626373626374,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":31.5789473684211},"score":31.5789473684211,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":52.1739130434783},"score":52.1739130434783,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":9.83184113865932},"score":9.83184113865932,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":23.7489557226399},"score":23.7489557226399,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":59.2592592592593},"score":59.2592592592593,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":39.778325123152705},"score":39.778325123152705,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":15.8333333333333},"score":15.8333333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":4.3859649122807},"score":4.3859649122807,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":34.276455026455},"score":34.276455026455,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":47.6764968722073},"score":47.6764968722073,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":51.685393258427005},"score":51.685393258427005,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":57.9710144927536},"score":57.9710144927536,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":34.2222222222222},"score":34.2222222222222,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":27.7044854881267},"score":27.7044854881267,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":35.342465753424705},"score":35.342465753424705,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":7.000000000000001},"score":7.000000000000001,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":36.1240310077519},"score":36.1240310077519,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":28.915662650602396},"score":28.915662650602396,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":27.7777777777778},"score":27.7777777777778,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank30","cate":"Submitted (repo)","github_url":"https://github.com/huangpengtao00-dotcom/memfusion-memory","system_name":"memfusion","target_version_label":"memfusion","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":31,"submission_id":"excel_academic_all_submission_32","result_id":"excel_academic_all_result_32","job_id":"excel_academic_all_job_32","user_id":"excel_academic_public","target_version_label":"PI-mem","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":40.031412456407104,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":40.031412456407104,"llm_score":40.031412456407104},"score":40.031412456407104,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":50.2123831267667},"score":50.2123831267667,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":37.6429206436794},"score":37.6429206436794,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":24.9204219553057},"score":24.9204219553057,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":32.7561423251078},"score":32.7561423251078,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":43.419765166340504},"score":43.419765166340504,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":28.191937424789398},"score":28.191937424789398,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":32.9032258064516},"score":32.9032258064516,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":51.0459512307338},"score":51.0459512307338,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":44.6601941747573},"score":44.6601941747573,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":43.3778625954198},"score":43.3778625954198,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":39.0605852363068},"score":39.0605852363068,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":26.3736263736264},"score":26.3736263736264,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":23.6842105263158},"score":23.6842105263158,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":57.9710144927536},"score":57.9710144927536,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":20.9308179194543},"score":20.9308179194543,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":35.5677885189163},"score":35.5677885189163,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":62.962962962963},"score":62.962962962963,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":45.381773399014804},"score":45.381773399014804,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":19.7916666666667},"score":19.7916666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":9.649122807017541},"score":9.649122807017541,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":33.942760942760906},"score":33.942760942760906,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":45.0067024128686},"score":45.0067024128686,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":44.9438202247191},"score":44.9438202247191,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":39.130434782608695},"score":39.130434782608695,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":31.7777777777778},"score":31.7777777777778,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":22.6912928759894},"score":22.6912928759894,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":35.342465753424705},"score":35.342465753424705,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":5},"score":5,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":33.7015503875969},"score":33.7015503875969,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":36.144578313253},"score":36.144578313253,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":29.166666666666703},"score":29.166666666666703,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank32","cate":"Submitted (API)","github_url":"https://github.com/fendss/pi-mem","system_name":"PI-mem","target_version_label":"PI-mem","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":32,"submission_id":"excel_academic_all_submission_33","result_id":"excel_academic_all_result_33","job_id":"excel_academic_all_job_33","user_id":"excel_academic_public","target_version_label":"mem0-BQE","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":39.8262114531271,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":39.8262114531271,"llm_score":39.8262114531271},"score":39.8262114531271,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":49.2953359425962},"score":49.2953359425962,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":43.1098651811854},"score":43.1098651811854,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":18.0031660845614},"score":18.0031660845614,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":34.047609478644},"score":34.047609478644,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":56.890084801043706},"score":56.890084801043706,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":22.8399518652226},"score":22.8399518652226,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":42.5806451612903},"score":42.5806451612903,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":50.317287784679095},"score":50.317287784679095,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":34.9514563106796},"score":34.9514563106796,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":42.0801526717557},"score":42.0801526717557,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":45.0074722485588},"score":45.0074722485588,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":24.1758241758242},"score":24.1758241758242,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":26.315789473684198},"score":26.315789473684198,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":55.072463768115895},"score":55.072463768115895,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":14.8011773579955},"score":14.8011773579955,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":24.3516529418785},"score":24.3516529418785,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":51.851851851851805},"score":51.851851851851805,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":47.9064039408867},"score":47.9064039408867,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":16.25},"score":16.25,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":7.01754385964912},"score":7.01754385964912,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":38.5158128908129},"score":38.5158128908129,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":54.881590705987506},"score":54.881590705987506,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":56.7415730337079},"score":56.7415730337079,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":62.3188405797101},"score":62.3188405797101,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":26.222222222222204},"score":26.222222222222204,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":17.4142480211082},"score":17.4142480211082,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":26.3013698630137},"score":26.3013698630137,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":5},"score":5,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":29.922480620155},"score":29.922480620155,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":50.6024096385542},"score":50.6024096385542,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank33","cate":"Submitted (repo)","github_url":"https://github.com/Cyrus580529/mem0-agent-memory-challenge","system_name":"mem0-BQE","target_version_label":"mem0-BQE","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":33,"submission_id":"excel_academic_all_submission_34","result_id":"excel_academic_all_result_34","job_id":"excel_academic_all_job_34","user_id":"excel_academic_public","target_version_label":"M-flow","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":39.611231101511905,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":39.611231101511905,"llm_score":39.611231101511905},"score":39.611231101511905,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":61.1415525114155},"score":61.1415525114155,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":42.3368740515933},"score":42.3368740515933,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":35.4203935599284},"score":35.4203935599284,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":28.3819628647215},"score":28.3819628647215,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":58.317025440313095},"score":58.317025440313095,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":3.47322720694646},"score":3.47322720694646,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":56.774193548387096},"score":56.774193548387096,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":62.6570048309179},"score":62.6570048309179,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":39.8058252427184},"score":39.8058252427184,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":40.458015267175604},"score":40.458015267175604,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":44.1426146010187},"score":44.1426146010187,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":36.2637362637363},"score":36.2637362637363,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":18.4210526315789},"score":18.4210526315789,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":60.8695652173913},"score":60.8695652173913,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":34.0909090909091},"score":34.0909090909091,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":24.060150375939802},"score":24.060150375939802,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":50},"score":50,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":45.812807881773395},"score":45.812807881773395,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":11.6666666666667},"score":11.6666666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":3.50877192982456},"score":3.50877192982456,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":16.6666666666667},"score":16.6666666666667,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":56.8364611260054},"score":56.8364611260054,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":56.1797752808989},"score":56.1797752808989,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":62.3188405797101},"score":62.3188405797101,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":0.22271714922049002},"score":0.22271714922049002,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":0.263852242744063},"score":0.263852242744063,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":0.8264462809917361},"score":0.8264462809917361,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":1},"score":1,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":19.5348837209302},"score":19.5348837209302,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":78.31325301204821},"score":78.31325301204821,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":31.944444444444397},"score":31.944444444444397,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank34","cate":"Submitted (repo)","github_url":"https://github.com/FlowElement-xinliuyuansu/m_flow","system_name":"M-flow","target_version_label":"M-flow","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":34,"submission_id":"excel_academic_all_submission_35","result_id":"excel_academic_all_result_35","job_id":"excel_academic_all_job_35","user_id":"excel_academic_public","target_version_label":"openProustian-agent-memory","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":39.0351987389544,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":39.0351987389544,"llm_score":39.0351987389544},"score":39.0351987389544,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":46.4092737551642},"score":46.4092737551642,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":35.9416569773171},"score":35.9416569773171,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":19.966351477979398},"score":19.966351477979398,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":30.731854352544},"score":30.731854352544,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":56.3030006523157},"score":56.3030006523157,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":26.47412755716},"score":26.47412755716,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":36.7741935483871},"score":36.7741935483871,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":48.1334828617437},"score":48.1334828617437,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":25.2427184466019},"score":25.2427184466019,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":35.9033078880407},"score":35.9033078880407,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":38.5153683328556},"score":38.5153683328556,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":21.978021978022},"score":21.978021978022,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":0},"score":0,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":59.4202898550725},"score":59.4202898550725,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":18.7214679260134},"score":18.7214679260134,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":22.0766201217329},"score":22.0766201217329,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":35.185185185185205},"score":35.185185185185205,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":41.687192118226605},"score":41.687192118226605,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":19.7916666666667},"score":19.7916666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":7.01754385964912},"score":7.01754385964912,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":38.7234848484848},"score":38.7234848484848,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":55.6858802502234},"score":55.6858802502234,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":60.1123595505618},"score":60.1123595505618,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":57.9710144927536},"score":57.9710144927536,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":28.6666666666667},"score":28.6666666666667,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":20.5804749340369},"score":20.5804749340369,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.6986301369863},"score":33.6986301369863,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":31.4728682170543},"score":31.4728682170543,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":42.1686746987952},"score":42.1686746987952,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":30.555555555555603},"score":30.555555555555603,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank35","cate":"Submitted (repo)","github_url":"https://github.com/openProustian/openProustian-agent-memory","system_name":"openProustian-agent-memory","target_version_label":"openProustian-agent-memory","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":35,"submission_id":"excel_academic_all_submission_36","result_id":"excel_academic_all_result_36","job_id":"excel_academic_all_job_36","user_id":"excel_academic_public","target_version_label":"just-a-BM25","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":38.4746671955284,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":38.4746671955284,"llm_score":38.4746671955284},"score":38.4746671955284,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":45.927728854098696},"score":45.927728854098696,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":32.655987580114996},"score":32.655987580114996,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":19.859726836471},"score":19.859726836471,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":30.247090074676304},"score":30.247090074676304,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":55.365296803652996},"score":55.365296803652996,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":26.5643802647413},"score":26.5643802647413,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":33.5483870967742},"score":33.5483870967742,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":47.3341672417759},"score":47.3341672417759,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":28.155339805825204},"score":28.155339805825204,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":39.774173027989804},"score":39.774173027989804,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":34.1600947628112},"score":34.1600947628112,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":20.8791208791209},"score":20.8791208791209,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":18.4210526315789},"score":18.4210526315789,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":42.028985507246404},"score":42.028985507246404,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":17.9784861603043},"score":17.9784861603043,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":20.3126864781},"score":20.3126864781,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":42.592592592592595},"score":42.592592592592595,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":41.564039408866996},"score":41.564039408866996,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.2083333333333},"score":20.2083333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":6.14035087719298},"score":6.14035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":37.760882635882595},"score":37.760882635882595,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":54.13315460232349},"score":54.13315460232349,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":55.6179775280899},"score":55.6179775280899,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":58.695652173913004},"score":58.695652173913004,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30.666666666666696},"score":30.666666666666696,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.1081794195251},"score":21.1081794195251,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":31.2328767123288},"score":31.2328767123288,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":8},"score":8,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":34.3798449612403},"score":34.3798449612403,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":34.9397590361446},"score":34.9397590361446,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":31.944444444444397},"score":31.944444444444397,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank36","cate":"Submitted (repo)","github_url":"https://github.com/imlrz/just-a-BM25","system_name":"just-a-BM25","target_version_label":"just-a-BM25","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":36,"submission_id":"excel_academic_all_submission_37","result_id":"excel_academic_all_result_37","job_id":"excel_academic_all_job_37","user_id":"excel_academic_public","target_version_label":"agent-memory-service","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":38.0399505523615,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":38.0399505523615,"llm_score":38.0399505523615},"score":38.0399505523615,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":46.11556860187},"score":46.11556860187,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":37.394205440487696},"score":37.394205440487696,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":17.8332907402675},"score":17.8332907402675,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":33.670697118972996},"score":33.670697118972996,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":54.924983692107},"score":54.924983692107,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":24.3742478941035},"score":24.3742478941035,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":40},"score":40,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":47.243041177823805},"score":47.243041177823805,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":36.893203883495204},"score":36.893203883495204,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":35.324427480916},"score":35.324427480916,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":39.800987071785},"score":39.800987071785,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":17.5824175824176},"score":17.5824175824176,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":18.4210526315789},"score":18.4210526315789,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":50.7246376811594},"score":50.7246376811594,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":15.018201495474202},"score":15.018201495474202,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":23.2617257429287},"score":23.2617257429287,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":46.296296296296305},"score":46.296296296296305,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":46.1822660098522},"score":46.1822660098522,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":19.5833333333333},"score":19.5833333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":9.649122807017541},"score":9.649122807017541,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":36.980880230880196},"score":36.980880230880196,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":52.993744414656},"score":52.993744414656,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":53.9325842696629},"score":53.9325842696629,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":60.1449275362319},"score":60.1449275362319,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":27.111111111111104},"score":27.111111111111104,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":17.4142480211082},"score":17.4142480211082,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":29.8630136986301},"score":29.8630136986301,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":7.000000000000001},"score":7.000000000000001,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":31.434108527131798},"score":31.434108527131798,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":45.783132530120504},"score":45.783132530120504,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank37","cate":"Submitted (repo)","github_url":"https://github.com/dddgzeth/agent-memory-service","system_name":"agent-memory-service","target_version_label":"agent-memory-service","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":37,"submission_id":"excel_academic_all_submission_38","result_id":"excel_academic_all_result_38","job_id":"excel_academic_all_job_38","user_id":"excel_academic_public","target_version_label":"LightMem-Ego-lite","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":38.032215793655496,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":38.032215793655496,"llm_score":38.032215793655496},"score":38.032215793655496,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":45.6783539899978},"score":45.6783539899978,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":34.9696537234474},"score":34.9696537234474,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":20.991211630746502},"score":20.991211630746502,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":32.560318034455996},"score":32.560318034455996,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":56.2133072407045},"score":56.2133072407045,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":24.067388688327302},"score":24.067388688327302,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":36.1290322580645},"score":36.1290322580645,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":46.6838624338624},"score":46.6838624338624,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":37.864077669902905},"score":37.864077669902905,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":42.010178117048405},"score":42.010178117048405,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":36.239391856964},"score":36.239391856964,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":27.472527472527503},"score":27.472527472527503,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":18.4210526315789},"score":18.4210526315789,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":47.8260869565217},"score":47.8260869565217,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":18.2522464908829},"score":18.2522464908829,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":23.188626327724098},"score":23.188626327724098,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":51.851851851851805},"score":51.851851851851805,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":46.1206896551724},"score":46.1206896551724,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":15.625},"score":15.625,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":6.14035087719298},"score":6.14035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":36.878998316498304},"score":36.878998316498304,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":55.2949061662198},"score":55.2949061662198,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":57.86516853932579},"score":57.86516853932579,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":58.695652173913004},"score":58.695652173913004,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":26.222222222222204},"score":26.222222222222204,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":17.6781002638522},"score":17.6781002638522,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":30.4109589041096},"score":30.4109589041096,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":32.2480620155039},"score":32.2480620155039,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":38.5542168674699},"score":38.5542168674699,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank38","cate":"Evaluated","github_url":"https://github.com/zjunlp/LightMem-Ego","system_name":"LightMem-Ego-lite","target_version_label":"LightMem-Ego-lite","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":38,"submission_id":"excel_academic_all_submission_39","result_id":"excel_academic_all_result_39","job_id":"excel_academic_all_job_39","user_id":"excel_academic_public","target_version_label":"ConstructMemory-AMC","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":37.8077431058761,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":37.8077431058761,"llm_score":37.8077431058761},"score":37.8077431058761,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":43.234099804305295},"score":43.234099804305295,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":27.580147891225298},"score":27.580147891225298,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":17.9726410540364},"score":17.9726410540364,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":33.798997936929},"score":33.798997936929,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":55.52837573385519},"score":55.52837573385519,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":26.708784596871197},"score":26.708784596871197,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":47.0967741935484},"score":47.0967741935484,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":44.870859213250505},"score":44.870859213250505,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":27.184466019417503},"score":27.184466019417503,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":28.4319338422392},"score":28.4319338422392,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":28.990352224647598},"score":28.990352224647598,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":10.989010989011},"score":10.989010989011,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":10.526315789473701},"score":10.526315789473701,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":42.028985507246404},"score":42.028985507246404,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":15.7989800603437},"score":15.7989800603437,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":17.832378565461298},"score":17.832378565461298,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":42.592592592592595},"score":42.592592592592595,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":46.551724137931},"score":46.551724137931,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.4166666666667},"score":20.4166666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":11.4035087719298},"score":11.4035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":33.2037037037037},"score":33.2037037037037,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":52.74798927613939},"score":52.74798927613939,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":56.7415730337079},"score":56.7415730337079,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":63.0434782608696},"score":63.0434782608696,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":29.777777777777796},"score":29.777777777777796,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":22.4274406332454},"score":22.4274406332454,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.1506849315069},"score":33.1506849315069,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":5},"score":5,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":28.798449612403097},"score":28.798449612403097,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":53.0120481927711},"score":53.0120481927711,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":40.2777777777778},"score":40.2777777777778,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank39","cate":"Submitted (repo)","github_url":"https://github.com/UnrealdDei/construct-memory-amc","system_name":"ConstructMemory-AMC","target_version_label":"ConstructMemory-AMC","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":39,"submission_id":"excel_academic_all_submission_40","result_id":"excel_academic_all_result_40","job_id":"excel_academic_all_job_40","user_id":"excel_academic_public","target_version_label":"M2A","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":36.9241624564849,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":36.9241624564849,"llm_score":36.9241624564849},"score":36.9241624564849,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":41.3293650793651},"score":41.3293650793651,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":27.0828242319138},"score":27.0828242319138,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":15.027543515915601},"score":15.027543515915601,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":32.9786162113748},"score":32.9786162113748,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":55.0391389432485},"score":55.0391389432485,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.996389891696698},"score":27.996389891696698,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":43.8709677419355},"score":43.8709677419355,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":42.4209224752703},"score":42.4209224752703,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":33.0097087378641},"score":33.0097087378641,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":38.7150127226463},"score":38.7150127226463,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":28.264144599034204},"score":28.264144599034204,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":16.4835164835165},"score":16.4835164835165,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":10.526315789473701},"score":10.526315789473701,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":44.927536231884105},"score":44.927536231884105,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":12.3975141020596},"score":12.3975141020596,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":18.9879460556152},"score":18.9879460556152,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":42.592592592592595},"score":42.592592592592595,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":47.0443349753695},"score":47.0443349753695,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":24.5833333333333},"score":24.5833333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":7.89473684210526},"score":7.89473684210526,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":33.0489718614719},"score":33.0489718614719,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":53.4182305630027},"score":53.4182305630027,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":57.86516853932579},"score":57.86516853932579,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":59.4202898550725},"score":59.4202898550725,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30.444444444444503},"score":30.444444444444503,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":26.1213720316623},"score":26.1213720316623,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":31.7808219178082},"score":31.7808219178082,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":9},"score":9,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":32.4418604651163},"score":32.4418604651163,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":51.8072289156627},"score":51.8072289156627,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":34.7222222222222},"score":34.7222222222222,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank40","cate":"Evaluated","github_url":"https://github.com/Little-Fridge/M2A","system_name":"M2A","target_version_label":"M2A","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":40,"submission_id":"excel_academic_all_submission_41","result_id":"excel_academic_all_result_41","job_id":"excel_academic_all_job_41","user_id":"excel_academic_public","target_version_label":"memoria_github_fireme_v3","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":36.5705052436112,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":36.5705052436112,"llm_score":36.5705052436112},"score":36.5705052436112,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":42.9306642748424},"score":42.9306642748424,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":36.2109798395838},"score":36.2109798395838,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":20.5981486214044},"score":20.5981486214044,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":19.8239547808513},"score":19.8239547808513,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":52.57664709719509},"score":52.57664709719509,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":26.7870036101083},"score":26.7870036101083,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":33.5483870967742},"score":33.5483870967742,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":43.921813894639996},"score":43.921813894639996,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":31.0679611650485},"score":31.0679611650485,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":37.8784987277354},"score":37.8784987277354,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":38.6469197186515},"score":38.6469197186515,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":25.274725274725302},"score":25.274725274725302,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":18.4210526315789},"score":18.4210526315789,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":44.927536231884105},"score":44.927536231884105,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":18.6247212383576},"score":18.6247212383576,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":20.032820145602102},"score":20.032820145602102,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":40.7407407407407},"score":40.7407407407407,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":23.7684729064039},"score":23.7684729064039,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20},"score":20,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":2.63157894736842},"score":2.63157894736842,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":39.1438492063492},"score":39.1438492063492,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":50.3127792672029},"score":50.3127792672029,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":56.7415730337079},"score":56.7415730337079,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":58.695652173913004},"score":58.695652173913004,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30.666666666666696},"score":30.666666666666696,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":20.844327176781},"score":20.844327176781,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":32.054794520548},"score":32.054794520548,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":30.2325581395349},"score":30.2325581395349,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":33.734939759036095},"score":33.734939759036095,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank41","cate":"Submitted (repo)","github_url":"https://github.com/FairmeHIT/memoria","system_name":"memoria_github_fireme_v3","target_version_label":"memoria_github_fireme_v3","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":41,"submission_id":"excel_academic_all_submission_42","result_id":"excel_academic_all_result_42","job_id":"excel_academic_all_job_42","user_id":"excel_academic_public","target_version_label":"Infini-memory","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":36.172326546274306,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":36.172326546274306,"llm_score":36.172326546274306},"score":36.172326546274306,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":43.2926179604262},"score":43.2926179604262,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":32.163571549004},"score":32.163571549004,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":13.1495442541954},"score":13.1495442541954,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":14.3898151656772},"score":14.3898151656772,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":40.2804957599478},"score":40.2804957599478,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.4007220216607},"score":27.4007220216607,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":51.61290322580649},"score":51.61290322580649,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":45.1260064412238},"score":45.1260064412238,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":18.446601941747602},"score":18.446601941747602,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":10.9287531806616},"score":10.9287531806616,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":34.1184951626378},"score":34.1184951626378,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":15.3846153846154},"score":15.3846153846154,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":7.89473684210526},"score":7.89473684210526,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":60.8695652173913},"score":60.8695652173913,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":12.2946182605273},"score":12.2946182605273,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":13.162370211242399},"score":13.162370211242399,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":24.0740740740741},"score":24.0740740740741,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":15.640394088670002},"score":15.640394088670002,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":12.0833333333333},"score":12.0833333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":8.7719298245614},"score":8.7719298245614,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":20.832671957671998},"score":20.832671957671998,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":41.5102770330652},"score":41.5102770330652,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":39.325842696629195},"score":39.325842696629195,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":36.9565217391304},"score":36.9565217391304,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":33.5555555555556},"score":33.5555555555556,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":22.4274406332454},"score":22.4274406332454,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":30.958904109589003},"score":30.958904109589003,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":28.139534883720902},"score":28.139534883720902,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":68.6746987951807},"score":68.6746987951807,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":31.944444444444397},"score":31.944444444444397,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank42","cate":"Evaluated","github_url":"https://github.com/infinigence/Infini-Memory","system_name":"Infini-memory","target_version_label":"Infini-memory","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":42,"submission_id":"excel_academic_all_submission_43","result_id":"excel_academic_all_result_43","job_id":"excel_academic_all_job_43","user_id":"excel_academic_public","target_version_label":"TraceMem v0.2.0","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":35.5123788064107,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":35.5123788064107,"llm_score":35.5123788064107},"score":35.5123788064107,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":38.1254348771472},"score":38.1254348771472,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":33.3462524661311},"score":33.3462524661311,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":13.121503819178201},"score":13.121503819178201,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":26.191586191586204},"score":26.191586191586204,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":52.9354207436399},"score":52.9354207436399,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.942238267148},"score":27.942238267148,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":40},"score":40,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":38.83802047389},"score":38.83802047389,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":35.9223300970874},"score":35.9223300970874,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":39.7423664122137},"score":39.7423664122137,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":34.7626152379972},"score":34.7626152379972,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":17.5824175824176},"score":17.5824175824176,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":10.526315789473701},"score":10.526315789473701,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":49.2753623188406},"score":49.2753623188406,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":10.4027285845468},"score":10.4027285845468,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":17.1798543978995},"score":17.1798543978995,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":40.7407407407407},"score":40.7407407407407,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":29.8645320197044},"score":29.8645320197044,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.625},"score":20.625,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":13.157894736842099},"score":13.157894736842099,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":38.5287999037999},"score":38.5287999037999,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":50.8042895442359},"score":50.8042895442359,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":53.9325842696629},"score":53.9325842696629,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":58.695652173913004},"score":58.695652173913004,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":32.6666666666667},"score":32.6666666666667,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":20.5804749340369},"score":20.5804749340369,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":33.972602739726},"score":33.972602739726,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":32.093023255814},"score":32.093023255814,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":48.1927710843373},"score":48.1927710843373,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":30.555555555555603},"score":30.555555555555603,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank43","cate":"Submitted (repo)","github_url":"https://github.com/mhw-mathcode/TraceMem","system_name":"TraceMem v0.2.0","target_version_label":"TraceMem v0.2.0","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":43,"submission_id":"excel_academic_all_submission_44","result_id":"excel_academic_all_result_44","job_id":"excel_academic_all_job_44","user_id":"excel_academic_public","target_version_label":"LightMem-StuctMem","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":35.4764824714102,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":35.4764824714102,"llm_score":35.4764824714102},"score":35.4764824714102,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":38.7295607740813},"score":38.7295607740813,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":34.8644585898},"score":34.8644585898,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":19.4711360990431},"score":19.4711360990431,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":33.7920604299915},"score":33.7920604299915,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":55.6425309849967},"score":55.6425309849967,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":26.7298435619735},"score":26.7298435619735,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":44.5161290322581},"score":44.5161290322581,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":39.0423855532551},"score":39.0423855532551,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":45.6310679611651},"score":45.6310679611651,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":43.117048346055995},"score":43.117048346055995,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":36.6310326157525},"score":36.6310326157525,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":25.274725274725302},"score":25.274725274725302,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":13.157894736842099},"score":13.157894736842099,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":42.028985507246404},"score":42.028985507246404,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":15.256952643316298},"score":15.256952643316298,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":33.717030671917904},"score":33.717030671917904,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":59.2592592592593},"score":59.2592592592593,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":46.3054187192118},"score":46.3054187192118,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":13.3333333333333},"score":13.3333333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":9.649122807017541},"score":9.649122807017541,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":37.3267797017797},"score":37.3267797017797,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":55.3172475424486},"score":55.3172475424486,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":56.1797752808989},"score":56.1797752808989,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":31.555555555555497},"score":31.555555555555497,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":19.2612137203166},"score":19.2612137203166,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":31.506849315068504},"score":31.506849315068504,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":33.5852713178295},"score":33.5852713178295,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":51.8072289156627},"score":51.8072289156627,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":36.1111111111111},"score":36.1111111111111,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank44","cate":"Submitted (repo)","github_url":"https://github.com/zjunlp/LightMem","system_name":"LightMem-StuctMem","target_version_label":"LightMem-StuctMem","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":44,"submission_id":"excel_academic_all_submission_45","result_id":"excel_academic_all_result_45","job_id":"excel_academic_all_job_45","user_id":"excel_academic_public","target_version_label":"picoagent","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":34.6912212844114,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":34.6912212844114,"llm_score":34.6912212844114},"score":34.6912212844114,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":40.2429876060013},"score":40.2429876060013,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":31.292337530956598},"score":31.292337530956598,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":18.709501093222},"score":18.709501093222,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":26.150329555502},"score":26.150329555502,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":49.6575342465753},"score":49.6575342465753,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":24.007220216606502},"score":24.007220216606502,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":38.7096774193548},"score":38.7096774193548,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":41.1266390614217},"score":41.1266390614217,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":32.0388349514563},"score":32.0388349514563,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":24.4783715012723},"score":24.4783715012723,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":33.313498188286},"score":33.313498188286,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":19.7802197802198},"score":19.7802197802198,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":7.89473684210526},"score":7.89473684210526,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":52.1739130434783},"score":52.1739130434783,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":16.4434113865932},"score":16.4434113865932,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":19.6136173767753},"score":19.6136173767753,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":42.592592592592595},"score":42.592592592592595,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":35.0369458128079},"score":35.0369458128079,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":19.7916666666667},"score":19.7916666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":3.50877192982456},"score":3.50877192982456,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":39.102904040404},"score":39.102904040404,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":47.1179624664879},"score":47.1179624664879,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":49.438202247191},"score":49.438202247191,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":28.4444444444444},"score":28.4444444444444,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":17.1503957783641},"score":17.1503957783641,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":29.8630136986301},"score":29.8630136986301,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":5},"score":5,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":27.6744186046512},"score":27.6744186046512,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":39.7590361445783},"score":39.7590361445783,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":37.5},"score":37.5,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank45","cate":"Submitted (repo)","github_url":"https://github.com/Chusi-Truth/picoAgent","system_name":"picoagent","target_version_label":"picoagent","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":45,"submission_id":"excel_academic_all_submission_46","result_id":"excel_academic_all_result_46","job_id":"excel_academic_all_job_46","user_id":"excel_academic_public","target_version_label":"MemoryBear","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":34.304915891361,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":34.304915891361,"llm_score":34.304915891361},"score":34.304915891361,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":37.565557729941304},"score":37.565557729941304,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":33.9760711384384},"score":33.9760711384384,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":18.2443848141523},"score":18.2443848141523,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":28.2294238328721},"score":28.2294238328721,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":54.476516634050896},"score":54.476516634050896,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":25.9205776173285},"score":25.9205776173285,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":29.6774193548387},"score":29.6774193548387,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":38.0524499654935},"score":38.0524499654935,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":39.8058252427185},"score":39.8058252427185,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":42.5254452926209},"score":42.5254452926209,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":35.976622886639895},"score":35.976622886639895,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":23.0769230769231},"score":23.0769230769231,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":13.157894736842099},"score":13.157894736842099,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":39.130434782608695},"score":39.130434782608695,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":15.286386593204801},"score":15.286386593204801,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":25.1775271512114},"score":25.1775271512114,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":50},"score":50,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":36.1453201970443},"score":36.1453201970443,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":23.5416666666667},"score":23.5416666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":6.14035087719298},"score":6.14035087719298,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":43.416546416546396},"score":43.416546416546396,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":54.7922252010724},"score":54.7922252010724,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":51.685393258427005},"score":51.685393258427005,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":53.623188405797094},"score":53.623188405797094,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":26},"score":26,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":20.3166226912929},"score":20.3166226912929,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":35.0684931506849},"score":35.0684931506849,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":6},"score":6,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":33.0232558139535},"score":33.0232558139535,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":30.1204819277109},"score":30.1204819277109,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":29.166666666666703},"score":29.166666666666703,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank46","cate":"Submitted (repo)","github_url":"https://github.com/SuanmoSuanyangTechnology/MemoryBear","system_name":"MemoryBear","target_version_label":"MemoryBear","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":46,"submission_id":"excel_academic_all_submission_47","result_id":"excel_academic_all_result_47","job_id":"excel_academic_all_job_47","user_id":"excel_academic_public","target_version_label":"AML-HM","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":33.8654743762126,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":33.8654743762126,"llm_score":33.8654743762126},"score":33.8654743762126,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":36.982903892150496},"score":36.982903892150496,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":25.233748716298},"score":25.233748716298,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":13.263480705341198},"score":13.263480705341198,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":24.558525765422303},"score":24.558525765422303,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":51.46771037182},"score":51.46771037182,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":27.3164861612515},"score":27.3164861612515,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":45.9677419354839},"score":45.9677419354839,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":37.7741833448355},"score":37.7741833448355,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":29.126213592233004},"score":29.126213592233004,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":28.889949109414797},"score":28.889949109414797,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":26.8744319253657},"score":26.8744319253657,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":16.4835164835165},"score":16.4835164835165,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":10.526315789473701},"score":10.526315789473701,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":40.5797101449275},"score":40.5797101449275,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":10.3600944510035},"score":10.3600944510035,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":15.896885069817401},"score":15.896885069817401,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":42.592592592592595},"score":42.592592592592595,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":35.5295566502463},"score":35.5295566502463,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":14.7916666666667},"score":14.7916666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":3.50877192982456},"score":3.50877192982456,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":27.4344035594035},"score":27.4344035594035,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":50.6702412868633},"score":50.6702412868633,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":53.9325842696629},"score":53.9325842696629,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":53.623188405797094},"score":53.623188405797094,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":30},"score":30,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":23.4828496042216},"score":23.4828496042216,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":32.6027397260274},"score":32.6027397260274,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":7.000000000000001},"score":7.000000000000001,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":32.2480620155039},"score":32.2480620155039,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":56.927710843373504},"score":56.927710843373504,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank47","cate":"Submitted (repo)","github_url":"https://github.com/pkgunboat/aml-hm","system_name":"AML-HM","target_version_label":"AML-HM","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":47,"submission_id":"excel_academic_all_submission_48","result_id":"excel_academic_all_result_48","job_id":"excel_academic_all_job_48","user_id":"excel_academic_public","target_version_label":"LightMem","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":28.7661391597826,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":28.7661391597826,"llm_score":28.7661391597826},"score":28.7661391597826,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":29.784273755164197},"score":29.784273755164197,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":27.3674803530646},"score":27.3674803530646,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":14.6489621489621},"score":14.6489621489621,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":28.887338714924898},"score":28.887338714924898,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":54.093281148075704},"score":54.093281148075704,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":19.6510228640192},"score":19.6510228640192,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":52.258064516129},"score":52.258064516129,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":30.641333103289597},"score":30.641333103289597,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":20.3883495145631},"score":20.3883495145631,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":27.5095419847328},"score":27.5095419847328,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":29.4315272541079},"score":29.4315272541079,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":9.89010989010989},"score":9.89010989010989,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":5.26315789473684},"score":5.26315789473684,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":42.028985507246404},"score":42.028985507246404,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":13.8197723993179},"score":13.8197723993179,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":20.968194295262},"score":20.968194295262,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":27.7777777777778},"score":27.7777777777778,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":40.332512315270904},"score":40.332512315270904,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":13.9583333333333},"score":13.9583333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":4.3859649122807},"score":4.3859649122807,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":36.7171115921116},"score":36.7171115921116,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":54.2672028596962},"score":54.2672028596962,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":58.9887640449438},"score":58.9887640449438,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":53.623188405797094},"score":53.623188405797094,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":21.3333333333333},"score":21.3333333333333,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":15.039577836411599},"score":15.039577836411599,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":24.109589041095898},"score":24.109589041095898,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":4},"score":4,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":29.3798449612403},"score":29.3798449612403,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":68.6746987951807},"score":68.6746987951807,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank48","cate":"Evaluated","github_url":"https://github.com/zjunlp/LightMem","system_name":"LightMem","target_version_label":"LightMem","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":48,"submission_id":"excel_academic_all_submission_49","result_id":"excel_academic_all_result_49","job_id":"excel_academic_all_job_49","user_id":"excel_academic_public","target_version_label":"perseus-vault","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":25.516832706828403,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":25.516832706828403,"llm_score":25.516832706828403},"score":25.516832706828403,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":21.137067840835},"score":21.137067840835,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":18.4351919367094},"score":18.4351919367094,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":10.194295368714},"score":10.194295368714,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":18.5385400040572},"score":18.5385400040572,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":49.9836921069798},"score":49.9836921069798,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":26.6546329723225},"score":26.6546329723225,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":38.0645161290323},"score":38.0645161290323,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":21.6377674258109},"score":21.6377674258109,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":22.3300970873786},"score":22.3300970873786,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":10.0286259541985},"score":10.0286259541985,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":19.7772351210382},"score":19.7772351210382,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":9.89010989010989},"score":9.89010989010989,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":5.26315789473684},"score":5.26315789473684,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":28.9855072463768},"score":28.9855072463768,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":7.4351469237832895},"score":7.4351469237832895,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":16.530910609857997},"score":16.530910609857997,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":37.037037037037},"score":37.037037037037,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":23.6453201970443},"score":23.6453201970443,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":20.4166666666667},"score":20.4166666666667,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":5.26315789473684},"score":5.26315789473684,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":26.483826358826402},"score":26.483826358826402,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":48.1009830205541},"score":48.1009830205541,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":52.247191011236005},"score":52.247191011236005,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":55.072463768115895},"score":55.072463768115895,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":32.6666666666667},"score":32.6666666666667,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":21.1081794195251},"score":21.1081794195251,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":30.1369863013699},"score":30.1369863013699,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":5},"score":5,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":29.8449612403101},"score":29.8449612403101,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":43.3734939759036},"score":43.3734939759036,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":31.944444444444397},"score":31.944444444444397,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank49","cate":"Submitted (repo)","github_url":"https://github.com/lkhup/Memory","system_name":"perseus-vault","target_version_label":"perseus-vault","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":49,"submission_id":"excel_academic_all_submission_50","result_id":"excel_academic_all_result_50","job_id":"excel_academic_all_job_50","user_id":"excel_academic_public","target_version_label":"Aranea-Agents-Memory-Correct","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":24.5640670994114,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":24.5640670994114,"llm_score":24.5640670994114},"score":24.5640670994114,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":17.2514948901935},"score":17.2514948901935,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":17.6106556569379},"score":17.6106556569379,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":8.895916744753949},"score":8.895916744753949,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":18.505919367988298},"score":18.505919367988298,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":51.4514024787997},"score":51.4514024787997,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":29.043321299639004},"score":29.043321299639004,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":39.354838709677395},"score":39.354838709677395,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":17.4786346905912},"score":17.4786346905912,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":20.3883495145631},"score":20.3883495145631,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":10.95737913486},"score":10.95737913486,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":18.6849271272022},"score":18.6849271272022,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":9.89010989010989},"score":9.89010989010989,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":7.89473684210526},"score":7.89473684210526,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":27.536231884057997},"score":27.536231884057997,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":6.55540797586252},"score":6.55540797586252,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":13.7053944384771},"score":13.7053944384771,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":29.6296296296296},"score":29.6296296296296,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":24.014778325123203},"score":24.014778325123203,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":19.5833333333333},"score":19.5833333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":3.50877192982456},"score":3.50877192982456,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":28.3621933621934},"score":28.3621933621934,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":49.575513851653305},"score":49.575513851653305,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":55.0561797752809},"score":55.0561797752809,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":56.5217391304348},"score":56.5217391304348,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":33.3333333333333},"score":33.3333333333333,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":22.9551451187335},"score":22.9551451187335,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":36.164383561643795},"score":36.164383561643795,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":8},"score":8,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":28.953488372093002},"score":28.953488372093002,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":42.1686746987952},"score":42.1686746987952,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":36.1111111111111},"score":36.1111111111111,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank50","cate":"Submitted (repo)","github_url":"https://github.com/AmarsDing/aranea-agents","system_name":"Aranea-Agents-Memory-Correct","target_version_label":"Aranea-Agents-Memory-Correct","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"},{"rank":50,"submission_id":"excel_academic_all_submission_51","result_id":"excel_academic_all_result_51","job_id":"excel_academic_all_job_51","user_id":"excel_academic_public","target_version_label":"agent-memory-leaderboard","target_git_sha":null,"target_deployment":"excel-import-fixed-1-all","mode":"full","benchmark_type":"textual","leaderboard_track":"academic","average_score":23.5460304378932,"dataset_scores":[{"dataset_id":"leaderboard_suite","metrics":{"accuracy":23.5460304378932,"llm_score":23.5460304378932},"score":23.5460304378932,"status":"succeeded","total":1,"category_scores":[{"category":"A","group":"by_capability","metrics":{"accuracy":18.116655794738},"score":18.116655794738,"total":1},{"category":"B","group":"by_capability","metrics":{"accuracy":14.5648606156952},"score":14.5648606156952,"total":1},{"category":"C","group":"by_capability","metrics":{"accuracy":8.61004628446489},"score":8.61004628446489,"total":1},{"category":"D","group":"by_capability","metrics":{"accuracy":14.567463188152798},"score":14.567463188152798,"total":1},{"category":"E","group":"by_capability","metrics":{"accuracy":48.059360730593596},"score":48.059360730593596,"total":1},{"category":"G","group":"by_capability","metrics":{"accuracy":23.38146811071},"score":23.38146811071,"total":1},{"category":"H","group":"by_capability","metrics":{"accuracy":67.741935483871},"score":67.741935483871,"total":1},{"category":"A1","group":"by_capability_leaf","metrics":{"accuracy":18.5871865654474},"score":18.5871865654474,"total":1},{"category":"A2","group":"by_capability_leaf","metrics":{"accuracy":16.504854368932},"score":16.504854368932,"total":1},{"category":"A3","group":"by_capability_leaf","metrics":{"accuracy":8.86132315521629},"score":8.86132315521629,"total":1},{"category":"B1","group":"by_capability_leaf","metrics":{"accuracy":15.1073737618729},"score":15.1073737618729,"total":1},{"category":"B2","group":"by_capability_leaf","metrics":{"accuracy":6.5934065934065895},"score":6.5934065934065895,"total":1},{"category":"B3","group":"by_capability_leaf","metrics":{"accuracy":5.26315789473684},"score":5.26315789473684,"total":1},{"category":"B4","group":"by_capability_leaf","metrics":{"accuracy":31.8840579710145},"score":31.8840579710145,"total":1},{"category":"C1","group":"by_capability_leaf","metrics":{"accuracy":7.25829725829726},"score":7.25829725829726,"total":1},{"category":"C2","group":"by_capability_leaf","metrics":{"accuracy":11.188089270796},"score":11.188089270796,"total":1},{"category":"C3","group":"by_capability_leaf","metrics":{"accuracy":22.2222222222222},"score":22.2222222222222,"total":1},{"category":"D1","group":"by_capability_leaf","metrics":{"accuracy":21.551724137931},"score":21.551724137931,"total":1},{"category":"D2","group":"by_capability_leaf","metrics":{"accuracy":12.0833333333333},"score":12.0833333333333,"total":1},{"category":"D3","group":"by_capability_leaf","metrics":{"accuracy":1.75438596491228},"score":1.75438596491228,"total":1},{"category":"F1","group":"by_capability_leaf","metrics":{"accuracy":15.282227032227},"score":15.282227032227,"total":1},{"category":"E1","group":"by_capability_leaf","metrics":{"accuracy":45.4647006255585},"score":45.4647006255585,"total":1},{"category":"E2","group":"by_capability_leaf","metrics":{"accuracy":48.314606741573},"score":48.314606741573,"total":1},{"category":"E3","group":"by_capability_leaf","metrics":{"accuracy":55.072463768115895},"score":55.072463768115895,"total":1},{"category":"G1","group":"by_capability_leaf","metrics":{"accuracy":27.111111111111104},"score":27.111111111111104,"total":1},{"category":"G2","group":"by_capability_leaf","metrics":{"accuracy":18.9973614775726},"score":18.9973614775726,"total":1},{"category":"G3","group":"by_capability_leaf","metrics":{"accuracy":29.8630136986301},"score":29.8630136986301,"total":1},{"category":"G4","group":"by_capability_leaf","metrics":{"accuracy":3},"score":3,"total":1},{"category":"G5","group":"by_capability_leaf","metrics":{"accuracy":25.968992248062},"score":25.968992248062,"total":1},{"category":"H1","group":"by_capability_leaf","metrics":{"accuracy":90.3614457831325},"score":90.3614457831325,"total":1},{"category":"H2","group":"by_capability_leaf","metrics":{"accuracy":41.6666666666667},"score":41.6666666666667,"total":1}],"subdataset_scores":[],"raw_subdataset_scores":[],"reused_add":false,"start_from_step":"add"}],"metadata":{"evaluator_id":"official-benchmark-pipelines-v3","run_label":"excel-import-fixed-1-all-academic-rank51","cate":"Submitted (repo)","github_url":"https://github.com/cenjoy/agent-memory-leaderboard-2026","system_name":"agent-memory-leaderboard","target_version_label":"agent-memory-leaderboard","benchmark_type":"textual","leaderboard_track":"academic","score_scale":"0-100","dataset_suite_version":"public_suite_v3"},"token_cost":null,"evaluation_time_seconds":null,"name_embargoed":false,"name_embargo_until":null,"submitted_at":"2026-08-12T17:00:27+00:00"}]}