Alibaba-NLP/gme-Qwen2-VL-2B-Instruct
zetlyn/models-hf model hf Alibaba-NLP/gme-Qwen2-VL-2B-Instruct known 2024-12-21
https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct
Properties
| author | Alibaba-NLPreceipt
What the source handed over{
"_asked": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"_id": "676639e05b17ac358c84390d",
"author": "Alibaba-NLP",
"cardData": {
"base_model": [
"Qwen/Qwen2-VL-2B-Instruct"
],
"language": [
"en",
"zh"
],
"license": "apache-2.0",
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"tags": [
"mteb",
"sentence-transformers",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore"
]
},
"config": {
"architectures": [
"Qwen2VLForConditionalGeneration",
"GmeQwen2VL"
],
"auto_map": {
"AutoConfig": "modeling_gme_qwen2vl.GmeQwen2VLConfig",
"AutoModel": "modeling_gme_qwen2vl.GmeQwen2VL"
},
"model_type": "qwen2_vl",
"processor_config": {
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
},
"tokenizer_config": {
"bos_token": null,
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
"eos_token": "<|im_end|>",
"pad_token": "<|endoftext|>",
"unk_token": null
}
},
"createdAt": "2024-12-21T03:45:36.000Z",
"disabled": false,
"downloads": 10837,
"gated": false,
"id": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"lastModified": "2025-06-09T11:53:35.000Z",
"library_name": "sentence-transformers",
"likes": 135,
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"modelId": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"pipeline_tag": "sentence-similarity",
"private": false,
"safetensors": {
"parameters": {
"F32": 2208985600
},
"total": 2208985600
},
"sha": "9cfa6413f704a7c1cf5064d240748e10c876b286",
"siblings": [
{
"rfilename": ".gitattributes"
},
{
"rfilename": "1_Pooling/config.json"
},
{
"rfilename": "README.md"
},
{
"rfilename": "added_tokens.json"
},
{
"rfilename": "chat_template.json"
},
{
"rfilename": "config.json"
},
{
"rfilename": "config_sentence_transformers.json"
},
{
"rfilename": "custom_st.py"
},
{
"rfilename": "generation_config.json"
},
{
"rfilename": "gme_inference.py"
},
{
"rfilename": "images/gme_logo.png"
},
{
"rfilename": "merges.txt"
},
{
"rfilename": "model-00001-of-00003.safetensors"
},
{
"rfilename": "model-00002-of-00003.safetensors"
},
{
"rfilename": "model-00003-of-00003.safetensors"
},
{
"rfilename": "model.safetensors.index.json"
},
{
"rfilename": "modeling_gme_qwen2vl.py"
},
{
"rfilename": "modules.json"
},
{
"rfilename": "preprocessor_config.json"
},
{
"rfilename": "results.json"
},
{
"rfilename": "special_tokens_map.json"
},
{
"rfilename": "tokenizer.json"
},
{
"rfilename": "tokenizer_config.json"
},
{
"rfilename": "vocab.json"
}
],
"spaces": [
"mteb/leaderboard",
"tardellirs/model-pulse",
"maxpar1/leaderboard",
"kiedistv/leaderboard",
"qiaosong88/leaderboard",
"ArtemBolgov/leaderboard",
"malyvena/leaderboard",
"HuggingLLM/leaderboard",
"wangsh0111/leaderboard",
"kaotd/leaderboard"
],
"tags": [
"sentence-transformers",
"safetensors",
"qwen2_vl",
"image-text-to-text",
"mteb",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore",
"custom_code",
"en",
"zh",
"arxiv:2412.16855",
"base_model:Qwen/Qwen2-VL-2B-Instruct",
"base_model:finetune:Qwen/Qwen2-VL-2B-Instruct",
"license:apache-2.0",
"model-index",
"endpoints_compatible",
"region:us"
],
"transformersInfo": {
"auto_model": "AutoModelForMultimodalLM",
"pipeline_tag": "image-text-to-text",
"processor": "AutoProcessor"
},
"usedStorage": 8836021280,
"widgetData": [
{
"sentences": [
"That is a happy dog",
"That is a very happy person",
"Today is a sunny day"
],
"source_sentence": "That is a happy person"
}
]
} | ||||||
|---|---|---|---|---|---|---|---|
| downloads | 10837receipt
What the source handed over{
"_asked": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"_id": "676639e05b17ac358c84390d",
"author": "Alibaba-NLP",
"cardData": {
"base_model": [
"Qwen/Qwen2-VL-2B-Instruct"
],
"language": [
"en",
"zh"
],
"license": "apache-2.0",
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"tags": [
"mteb",
"sentence-transformers",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore"
]
},
"config": {
"architectures": [
"Qwen2VLForConditionalGeneration",
"GmeQwen2VL"
],
"auto_map": {
"AutoConfig": "modeling_gme_qwen2vl.GmeQwen2VLConfig",
"AutoModel": "modeling_gme_qwen2vl.GmeQwen2VL"
},
"model_type": "qwen2_vl",
"processor_config": {
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
},
"tokenizer_config": {
"bos_token": null,
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
"eos_token": "<|im_end|>",
"pad_token": "<|endoftext|>",
"unk_token": null
}
},
"createdAt": "2024-12-21T03:45:36.000Z",
"disabled": false,
"downloads": 10837,
"gated": false,
"id": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"lastModified": "2025-06-09T11:53:35.000Z",
"library_name": "sentence-transformers",
"likes": 135,
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"modelId": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"pipeline_tag": "sentence-similarity",
"private": false,
"safetensors": {
"parameters": {
"F32": 2208985600
},
"total": 2208985600
},
"sha": "9cfa6413f704a7c1cf5064d240748e10c876b286",
"siblings": [
{
"rfilename": ".gitattributes"
},
{
"rfilename": "1_Pooling/config.json"
},
{
"rfilename": "README.md"
},
{
"rfilename": "added_tokens.json"
},
{
"rfilename": "chat_template.json"
},
{
"rfilename": "config.json"
},
{
"rfilename": "config_sentence_transformers.json"
},
{
"rfilename": "custom_st.py"
},
{
"rfilename": "generation_config.json"
},
{
"rfilename": "gme_inference.py"
},
{
"rfilename": "images/gme_logo.png"
},
{
"rfilename": "merges.txt"
},
{
"rfilename": "model-00001-of-00003.safetensors"
},
{
"rfilename": "model-00002-of-00003.safetensors"
},
{
"rfilename": "model-00003-of-00003.safetensors"
},
{
"rfilename": "model.safetensors.index.json"
},
{
"rfilename": "modeling_gme_qwen2vl.py"
},
{
"rfilename": "modules.json"
},
{
"rfilename": "preprocessor_config.json"
},
{
"rfilename": "results.json"
},
{
"rfilename": "special_tokens_map.json"
},
{
"rfilename": "tokenizer.json"
},
{
"rfilename": "tokenizer_config.json"
},
{
"rfilename": "vocab.json"
}
],
"spaces": [
"mteb/leaderboard",
"tardellirs/model-pulse",
"maxpar1/leaderboard",
"kiedistv/leaderboard",
"qiaosong88/leaderboard",
"ArtemBolgov/leaderboard",
"malyvena/leaderboard",
"HuggingLLM/leaderboard",
"wangsh0111/leaderboard",
"kaotd/leaderboard"
],
"tags": [
"sentence-transformers",
"safetensors",
"qwen2_vl",
"image-text-to-text",
"mteb",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore",
"custom_code",
"en",
"zh",
"arxiv:2412.16855",
"base_model:Qwen/Qwen2-VL-2B-Instruct",
"base_model:finetune:Qwen/Qwen2-VL-2B-Instruct",
"license:apache-2.0",
"model-index",
"endpoints_compatible",
"region:us"
],
"transformersInfo": {
"auto_model": "AutoModelForMultimodalLM",
"pipeline_tag": "image-text-to-text",
"processor": "AutoProcessor"
},
"usedStorage": 8836021280,
"widgetData": [
{
"sentences": [
"That is a happy dog",
"That is a very happy person",
"Today is a sunny day"
],
"source_sentence": "That is a happy person"
}
]
} | ||||||
| gated | falsereceipt
What the source handed over{
"_asked": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"_id": "676639e05b17ac358c84390d",
"author": "Alibaba-NLP",
"cardData": {
"base_model": [
"Qwen/Qwen2-VL-2B-Instruct"
],
"language": [
"en",
"zh"
],
"license": "apache-2.0",
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"tags": [
"mteb",
"sentence-transformers",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore"
]
},
"config": {
"architectures": [
"Qwen2VLForConditionalGeneration",
"GmeQwen2VL"
],
"auto_map": {
"AutoConfig": "modeling_gme_qwen2vl.GmeQwen2VLConfig",
"AutoModel": "modeling_gme_qwen2vl.GmeQwen2VL"
},
"model_type": "qwen2_vl",
"processor_config": {
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
},
"tokenizer_config": {
"bos_token": null,
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
"eos_token": "<|im_end|>",
"pad_token": "<|endoftext|>",
"unk_token": null
}
},
"createdAt": "2024-12-21T03:45:36.000Z",
"disabled": false,
"downloads": 10837,
"gated": false,
"id": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"lastModified": "2025-06-09T11:53:35.000Z",
"library_name": "sentence-transformers",
"likes": 135,
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"modelId": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"pipeline_tag": "sentence-similarity",
"private": false,
"safetensors": {
"parameters": {
"F32": 2208985600
},
"total": 2208985600
},
"sha": "9cfa6413f704a7c1cf5064d240748e10c876b286",
"siblings": [
{
"rfilename": ".gitattributes"
},
{
"rfilename": "1_Pooling/config.json"
},
{
"rfilename": "README.md"
},
{
"rfilename": "added_tokens.json"
},
{
"rfilename": "chat_template.json"
},
{
"rfilename": "config.json"
},
{
"rfilename": "config_sentence_transformers.json"
},
{
"rfilename": "custom_st.py"
},
{
"rfilename": "generation_config.json"
},
{
"rfilename": "gme_inference.py"
},
{
"rfilename": "images/gme_logo.png"
},
{
"rfilename": "merges.txt"
},
{
"rfilename": "model-00001-of-00003.safetensors"
},
{
"rfilename": "model-00002-of-00003.safetensors"
},
{
"rfilename": "model-00003-of-00003.safetensors"
},
{
"rfilename": "model.safetensors.index.json"
},
{
"rfilename": "modeling_gme_qwen2vl.py"
},
{
"rfilename": "modules.json"
},
{
"rfilename": "preprocessor_config.json"
},
{
"rfilename": "results.json"
},
{
"rfilename": "special_tokens_map.json"
},
{
"rfilename": "tokenizer.json"
},
{
"rfilename": "tokenizer_config.json"
},
{
"rfilename": "vocab.json"
}
],
"spaces": [
"mteb/leaderboard",
"tardellirs/model-pulse",
"maxpar1/leaderboard",
"kiedistv/leaderboard",
"qiaosong88/leaderboard",
"ArtemBolgov/leaderboard",
"malyvena/leaderboard",
"HuggingLLM/leaderboard",
"wangsh0111/leaderboard",
"kaotd/leaderboard"
],
"tags": [
"sentence-transformers",
"safetensors",
"qwen2_vl",
"image-text-to-text",
"mteb",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore",
"custom_code",
"en",
"zh",
"arxiv:2412.16855",
"base_model:Qwen/Qwen2-VL-2B-Instruct",
"base_model:finetune:Qwen/Qwen2-VL-2B-Instruct",
"license:apache-2.0",
"model-index",
"endpoints_compatible",
"region:us"
],
"transformersInfo": {
"auto_model": "AutoModelForMultimodalLM",
"pipeline_tag": "image-text-to-text",
"processor": "AutoProcessor"
},
"usedStorage": 8836021280,
"widgetData": [
{
"sentences": [
"That is a happy dog",
"That is a very happy person",
"Today is a sunny day"
],
"source_sentence": "That is a happy person"
}
]
} | ||||||
| licence | apache-2.0receipt
What the source handed over{
"_asked": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"_id": "676639e05b17ac358c84390d",
"author": "Alibaba-NLP",
"cardData": {
"base_model": [
"Qwen/Qwen2-VL-2B-Instruct"
],
"language": [
"en",
"zh"
],
"license": "apache-2.0",
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"tags": [
"mteb",
"sentence-transformers",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore"
]
},
"config": {
"architectures": [
"Qwen2VLForConditionalGeneration",
"GmeQwen2VL"
],
"auto_map": {
"AutoConfig": "modeling_gme_qwen2vl.GmeQwen2VLConfig",
"AutoModel": "modeling_gme_qwen2vl.GmeQwen2VL"
},
"model_type": "qwen2_vl",
"processor_config": {
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
},
"tokenizer_config": {
"bos_token": null,
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
"eos_token": "<|im_end|>",
"pad_token": "<|endoftext|>",
"unk_token": null
}
},
"createdAt": "2024-12-21T03:45:36.000Z",
"disabled": false,
"downloads": 10837,
"gated": false,
"id": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"lastModified": "2025-06-09T11:53:35.000Z",
"library_name": "sentence-transformers",
"likes": 135,
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"modelId": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"pipeline_tag": "sentence-similarity",
"private": false,
"safetensors": {
"parameters": {
"F32": 2208985600
},
"total": 2208985600
},
"sha": "9cfa6413f704a7c1cf5064d240748e10c876b286",
"siblings": [
{
"rfilename": ".gitattributes"
},
{
"rfilename": "1_Pooling/config.json"
},
{
"rfilename": "README.md"
},
{
"rfilename": "added_tokens.json"
},
{
"rfilename": "chat_template.json"
},
{
"rfilename": "config.json"
},
{
"rfilename": "config_sentence_transformers.json"
},
{
"rfilename": "custom_st.py"
},
{
"rfilename": "generation_config.json"
},
{
"rfilename": "gme_inference.py"
},
{
"rfilename": "images/gme_logo.png"
},
{
"rfilename": "merges.txt"
},
{
"rfilename": "model-00001-of-00003.safetensors"
},
{
"rfilename": "model-00002-of-00003.safetensors"
},
{
"rfilename": "model-00003-of-00003.safetensors"
},
{
"rfilename": "model.safetensors.index.json"
},
{
"rfilename": "modeling_gme_qwen2vl.py"
},
{
"rfilename": "modules.json"
},
{
"rfilename": "preprocessor_config.json"
},
{
"rfilename": "results.json"
},
{
"rfilename": "special_tokens_map.json"
},
{
"rfilename": "tokenizer.json"
},
{
"rfilename": "tokenizer_config.json"
},
{
"rfilename": "vocab.json"
}
],
"spaces": [
"mteb/leaderboard",
"tardellirs/model-pulse",
"maxpar1/leaderboard",
"kiedistv/leaderboard",
"qiaosong88/leaderboard",
"ArtemBolgov/leaderboard",
"malyvena/leaderboard",
"HuggingLLM/leaderboard",
"wangsh0111/leaderboard",
"kaotd/leaderboard"
],
"tags": [
"sentence-transformers",
"safetensors",
"qwen2_vl",
"image-text-to-text",
"mteb",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore",
"custom_code",
"en",
"zh",
"arxiv:2412.16855",
"base_model:Qwen/Qwen2-VL-2B-Instruct",
"base_model:finetune:Qwen/Qwen2-VL-2B-Instruct",
"license:apache-2.0",
"model-index",
"endpoints_compatible",
"region:us"
],
"transformersInfo": {
"auto_model": "AutoModelForMultimodalLM",
"pipeline_tag": "image-text-to-text",
"processor": "AutoProcessor"
},
"usedStorage": 8836021280,
"widgetData": [
{
"sentences": [
"That is a happy dog",
"That is a very happy person",
"Today is a sunny day"
],
"source_sentence": "That is a happy person"
}
]
} | ||||||
| likes | 135receipt
What the source handed over{
"_asked": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"_id": "676639e05b17ac358c84390d",
"author": "Alibaba-NLP",
"cardData": {
"base_model": [
"Qwen/Qwen2-VL-2B-Instruct"
],
"language": [
"en",
"zh"
],
"license": "apache-2.0",
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"tags": [
"mteb",
"sentence-transformers",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore"
]
},
"config": {
"architectures": [
"Qwen2VLForConditionalGeneration",
"GmeQwen2VL"
],
"auto_map": {
"AutoConfig": "modeling_gme_qwen2vl.GmeQwen2VLConfig",
"AutoModel": "modeling_gme_qwen2vl.GmeQwen2VL"
},
"model_type": "qwen2_vl",
"processor_config": {
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
},
"tokenizer_config": {
"bos_token": null,
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
"eos_token": "<|im_end|>",
"pad_token": "<|endoftext|>",
"unk_token": null
}
},
"createdAt": "2024-12-21T03:45:36.000Z",
"disabled": false,
"downloads": 10837,
"gated": false,
"id": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"lastModified": "2025-06-09T11:53:35.000Z",
"library_name": "sentence-transformers",
"likes": 135,
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"modelId": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"pipeline_tag": "sentence-similarity",
"private": false,
"safetensors": {
"parameters": {
"F32": 2208985600
},
"total": 2208985600
},
"sha": "9cfa6413f704a7c1cf5064d240748e10c876b286",
"siblings": [
{
"rfilename": ".gitattributes"
},
{
"rfilename": "1_Pooling/config.json"
},
{
"rfilename": "README.md"
},
{
"rfilename": "added_tokens.json"
},
{
"rfilename": "chat_template.json"
},
{
"rfilename": "config.json"
},
{
"rfilename": "config_sentence_transformers.json"
},
{
"rfilename": "custom_st.py"
},
{
"rfilename": "generation_config.json"
},
{
"rfilename": "gme_inference.py"
},
{
"rfilename": "images/gme_logo.png"
},
{
"rfilename": "merges.txt"
},
{
"rfilename": "model-00001-of-00003.safetensors"
},
{
"rfilename": "model-00002-of-00003.safetensors"
},
{
"rfilename": "model-00003-of-00003.safetensors"
},
{
"rfilename": "model.safetensors.index.json"
},
{
"rfilename": "modeling_gme_qwen2vl.py"
},
{
"rfilename": "modules.json"
},
{
"rfilename": "preprocessor_config.json"
},
{
"rfilename": "results.json"
},
{
"rfilename": "special_tokens_map.json"
},
{
"rfilename": "tokenizer.json"
},
{
"rfilename": "tokenizer_config.json"
},
{
"rfilename": "vocab.json"
}
],
"spaces": [
"mteb/leaderboard",
"tardellirs/model-pulse",
"maxpar1/leaderboard",
"kiedistv/leaderboard",
"qiaosong88/leaderboard",
"ArtemBolgov/leaderboard",
"malyvena/leaderboard",
"HuggingLLM/leaderboard",
"wangsh0111/leaderboard",
"kaotd/leaderboard"
],
"tags": [
"sentence-transformers",
"safetensors",
"qwen2_vl",
"image-text-to-text",
"mteb",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore",
"custom_code",
"en",
"zh",
"arxiv:2412.16855",
"base_model:Qwen/Qwen2-VL-2B-Instruct",
"base_model:finetune:Qwen/Qwen2-VL-2B-Instruct",
"license:apache-2.0",
"model-index",
"endpoints_compatible",
"region:us"
],
"transformersInfo": {
"auto_model": "AutoModelForMultimodalLM",
"pipeline_tag": "image-text-to-text",
"processor": "AutoProcessor"
},
"usedStorage": 8836021280,
"widgetData": [
{
"sentences": [
"That is a happy dog",
"That is a very happy person",
"Today is a sunny day"
],
"source_sentence": "That is a happy person"
}
]
} | ||||||
| task | sentence-similarityreceipt
What the source handed over{
"_asked": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"_id": "676639e05b17ac358c84390d",
"author": "Alibaba-NLP",
"cardData": {
"base_model": [
"Qwen/Qwen2-VL-2B-Instruct"
],
"language": [
"en",
"zh"
],
"license": "apache-2.0",
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"tags": [
"mteb",
"sentence-transformers",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore"
]
},
"config": {
"architectures": [
"Qwen2VLForConditionalGeneration",
"GmeQwen2VL"
],
"auto_map": {
"AutoConfig": "modeling_gme_qwen2vl.GmeQwen2VLConfig",
"AutoModel": "modeling_gme_qwen2vl.GmeQwen2VL"
},
"model_type": "qwen2_vl",
"processor_config": {
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
},
"tokenizer_config": {
"bos_token": null,
"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
"eos_token": "<|im_end|>",
"pad_token": "<|endoftext|>",
"unk_token": null
}
},
"createdAt": "2024-12-21T03:45:36.000Z",
"disabled": false,
"downloads": 10837,
"gated": false,
"id": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"lastModified": "2025-06-09T11:53:35.000Z",
"library_name": "sentence-transformers",
"likes": 135,
"model-index": [
{
"name": "external",
"results": [
{
"dataset": {
"config": "default",
"name": "MTEB AFQMC",
"revision": "b44c3b011063adb25877c13823db83bb193913c4",
"split": "validation",
"type": "C-MTEB/AFQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 61.03190209456061,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 65.38958681599493,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.54853383020948,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 65.25341659273157,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 67.34190190683134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ATEC",
"revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865",
"split": "test",
"type": "C-MTEB/ATEC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 50.83794357648487,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 54.03230997664373,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 55.2072028123375,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 54.032311102613264,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 55.05163232251946,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 53.81272176804127,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonCounterfactualClassification (en)",
"revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
"split": "test",
"type": "mteb/amazon_counterfactual"
},
"metrics": [
{
"type": "accuracy",
"value": 72.55223880597015,
"verified": false
},
{
"type": "ap",
"value": 35.01515316721116,
"verified": false
},
{
"type": "f1",
"value": 66.44086070814382,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AmazonPolarityClassification",
"revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
"split": "test",
"type": "mteb/amazon_polarity"
},
"metrics": [
{
"type": "accuracy",
"value": 96.7582,
"verified": false
},
{
"type": "ap",
"value": 95.5100924209288,
"verified": false
},
{
"type": "f1",
"value": 96.75713119357414,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB AmazonReviewsClassification (en)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 61.971999999999994,
"verified": false
},
{
"type": "f1",
"value": 60.50745575187704,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB AmazonReviewsClassification (zh)",
"revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
"split": "test",
"type": "mteb/amazon_reviews_multi"
},
"metrics": [
{
"type": "accuracy",
"value": 53.49,
"verified": false
},
{
"type": "f1",
"value": 51.576550662258434,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArguAna",
"revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
"split": "test",
"type": "mteb/arguana"
},
"metrics": [
{
"type": "map_at_1",
"value": 36.273,
"verified": false
},
{
"type": "map_at_10",
"value": 52.782,
"verified": false
},
{
"type": "map_at_100",
"value": 53.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 53.343,
"verified": false
},
{
"type": "map_at_3",
"value": 48.4,
"verified": false
},
{
"type": "map_at_5",
"value": 50.882000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 36.984,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.052,
"verified": false
},
{
"type": "mrr_at_100",
"value": 53.604,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 53.607000000000006,
"verified": false
},
{
"type": "mrr_at_3",
"value": 48.613,
"verified": false
},
{
"type": "mrr_at_5",
"value": 51.159,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 36.273,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 61.524,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.796,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.869,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 52.456,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.964000000000006,
"verified": false
},
{
"type": "precision_at_1",
"value": 36.273,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.926,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.989,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.408,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.05,
"verified": false
},
{
"type": "recall_at_1",
"value": 36.273,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.25999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.933,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.502,
"verified": false
},
{
"type": "recall_at_3",
"value": 64.225,
"verified": false
},
{
"type": "recall_at_5",
"value": 75.249,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringP2P",
"revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
"split": "test",
"type": "mteb/arxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 52.45236368396085,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ArxivClusteringS2S",
"revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
"split": "test",
"type": "mteb/arxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 46.83781937870832,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB AskUbuntuDupQuestions",
"revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
"split": "test",
"type": "mteb/askubuntudupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 60.653430349851746,
"verified": false
},
{
"type": "mrr",
"value": 74.28736314470387,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BIOSSES",
"revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
"split": "test",
"type": "mteb/biosses-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 89.18568151905953,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.25416218056225,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 86.47666922475281,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 87.04960508086356,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 86.73992823533615,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BQ",
"revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55",
"split": "test",
"type": "C-MTEB/BQ"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 75.7464284612374,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 77.63454068918787,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.71894224189296,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 77.58744810404339,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.63293552726073,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Banking77Classification",
"revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
"split": "test",
"type": "mteb/banking77"
},
"metrics": [
{
"type": "accuracy",
"value": 80.2435064935065,
"verified": false
},
{
"type": "f1",
"value": 79.44078343737895,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringP2P",
"revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
"split": "test",
"type": "mteb/biorxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.68220155432257,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB BiorxivClusteringS2S",
"revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
"split": "test",
"type": "mteb/biorxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 40.66615047758928,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringP2P",
"revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476",
"split": "test",
"type": "C-MTEB/CLSClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 44.23533333311907,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CLSClusteringS2S",
"revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f",
"split": "test",
"type": "C-MTEB/CLSClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 43.01114481307774,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv1",
"revision": "8d7f1e942507dac42dc58017c1a001c3717da7df",
"split": "test",
"type": "C-MTEB/CMedQAv1-reranking"
},
"metrics": [
{
"type": "map",
"value": 86.4349853821696,
"verified": false
},
{
"type": "mrr",
"value": 88.80150793650795,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CMedQAv2",
"revision": "23d186750531a14a0357ca22cd92d712fd512ea0",
"split": "test",
"type": "C-MTEB/CMedQAv2-reranking"
},
"metrics": [
{
"type": "map",
"value": 87.56417400982208,
"verified": false
},
{
"type": "mrr",
"value": 89.85813492063491,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackAndroidRetrieval",
"revision": "f46a197baaae43b4f621051089b82a364682dfeb",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 30.623,
"verified": false
},
{
"type": "map_at_10",
"value": 40.482,
"verified": false
},
{
"type": "map_at_100",
"value": 41.997,
"verified": false
},
{
"type": "map_at_1000",
"value": 42.135,
"verified": false
},
{
"type": "map_at_3",
"value": 37.754,
"verified": false
},
{
"type": "map_at_5",
"value": 39.031,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.482,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.311,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.212,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.27,
"verified": false
},
{
"type": "mrr_at_3",
"value": 44.157999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 45.145,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.482,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.142,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.834,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 54.164,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 42.309000000000005,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 43.485,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.482,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.455,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.378,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.188,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.172,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.705,
"verified": false
},
{
"type": "recall_at_1",
"value": 30.623,
"verified": false
},
{
"type": "recall_at_10",
"value": 56.77100000000001,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.034,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.629,
"verified": false
},
{
"type": "recall_at_3",
"value": 44.663,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.692,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackEnglishRetrieval",
"revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 27.941,
"verified": false
},
{
"type": "map_at_10",
"value": 38.437,
"verified": false
},
{
"type": "map_at_100",
"value": 39.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.753,
"verified": false
},
{
"type": "map_at_3",
"value": 35.388999999999996,
"verified": false
},
{
"type": "map_at_5",
"value": 37.113,
"verified": false
},
{
"type": "mrr_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "mrr_at_10",
"value": 43.864999999999995,
"verified": false
},
{
"type": "mrr_at_100",
"value": 44.533,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 44.581,
"verified": false
},
{
"type": "mrr_at_3",
"value": 41.55,
"verified": false
},
{
"type": "mrr_at_5",
"value": 42.942,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 44.330000000000005,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.61,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.712999999999994,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 39.834,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.016,
"verified": false
},
{
"type": "precision_at_1",
"value": 34.522000000000006,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.471,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.338,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.182,
"verified": false
},
{
"type": "precision_at_3",
"value": 19.363,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.898,
"verified": false
},
{
"type": "recall_at_1",
"value": 27.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.336,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.51100000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.636,
"verified": false
},
{
"type": "recall_at_3",
"value": 42.54,
"verified": false
},
{
"type": "recall_at_5",
"value": 48.392,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGamingRetrieval",
"revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 32.681,
"verified": false
},
{
"type": "map_at_10",
"value": 45.48,
"verified": false
},
{
"type": "map_at_100",
"value": 46.542,
"verified": false
},
{
"type": "map_at_1000",
"value": 46.604,
"verified": false
},
{
"type": "map_at_3",
"value": 42.076,
"verified": false
},
{
"type": "map_at_5",
"value": 44.076,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.492,
"verified": false
},
{
"type": "mrr_at_10",
"value": 48.746,
"verified": false
},
{
"type": "mrr_at_100",
"value": 49.485,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 49.517,
"verified": false
},
{
"type": "mrr_at_3",
"value": 45.998,
"verified": false
},
{
"type": "mrr_at_5",
"value": 47.681,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.492,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 51.779,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 56.294,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 57.58,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 45.856,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.968,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.492,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.620999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.189,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.135,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.773,
"verified": false
},
{
"type": "precision_at_5",
"value": 14.596,
"verified": false
},
{
"type": "recall_at_1",
"value": 32.681,
"verified": false
},
{
"type": "recall_at_10",
"value": 67.196,
"verified": false
},
{
"type": "recall_at_100",
"value": 87.027,
"verified": false
},
{
"type": "recall_at_1000",
"value": 96.146,
"verified": false
},
{
"type": "recall_at_3",
"value": 51.565000000000005,
"verified": false
},
{
"type": "recall_at_5",
"value": 59.123999999999995,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackGisRetrieval",
"revision": "5003b3064772da1887988e05400cf3806fe491f2",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 22.421,
"verified": false
},
{
"type": "map_at_10",
"value": 30.127,
"verified": false
},
{
"type": "map_at_100",
"value": 31.254,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.344,
"verified": false
},
{
"type": "map_at_3",
"value": 27.673,
"verified": false
},
{
"type": "map_at_5",
"value": 29.182,
"verified": false
},
{
"type": "mrr_at_1",
"value": 24.068,
"verified": false
},
{
"type": "mrr_at_10",
"value": 31.857000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 32.808,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 32.881,
"verified": false
},
{
"type": "mrr_at_3",
"value": 29.397,
"verified": false
},
{
"type": "mrr_at_5",
"value": 30.883,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 24.068,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.642,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.327,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.55,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 29.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.461,
"verified": false
},
{
"type": "precision_at_1",
"value": 24.068,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.390000000000001,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.873,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.109,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.692999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 9.107,
"verified": false
},
{
"type": "recall_at_1",
"value": 22.421,
"verified": false
},
{
"type": "recall_at_10",
"value": 46.846,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.409,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.06,
"verified": false
},
{
"type": "recall_at_3",
"value": 34.198,
"verified": false
},
{
"type": "recall_at_5",
"value": 40.437,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackMathematicaRetrieval",
"revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.494,
"verified": false
},
{
"type": "map_at_10",
"value": 24.4,
"verified": false
},
{
"type": "map_at_100",
"value": 25.719,
"verified": false
},
{
"type": "map_at_1000",
"value": 25.840000000000003,
"verified": false
},
{
"type": "map_at_3",
"value": 21.731,
"verified": false
},
{
"type": "map_at_5",
"value": 23.248,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.274,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.866000000000003,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.889,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.957,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.284999999999997,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.79,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.274,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.667,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 36.095,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.87,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 24.672,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 27.106,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.274,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.5969999999999995,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.04,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.141,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.023,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.494,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.401,
"verified": false
},
{
"type": "recall_at_100",
"value": 69.811,
"verified": false
},
{
"type": "recall_at_1000",
"value": 89.422,
"verified": false
},
{
"type": "recall_at_3",
"value": 27.834999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 33.774,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackPhysicsRetrieval",
"revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 26.15,
"verified": false
},
{
"type": "map_at_10",
"value": 36.012,
"verified": false
},
{
"type": "map_at_100",
"value": 37.377,
"verified": false
},
{
"type": "map_at_1000",
"value": 37.497,
"verified": false
},
{
"type": "map_at_3",
"value": 32.712,
"verified": false
},
{
"type": "map_at_5",
"value": 34.475,
"verified": false
},
{
"type": "mrr_at_1",
"value": 32.05,
"verified": false
},
{
"type": "mrr_at_10",
"value": 41.556,
"verified": false
},
{
"type": "mrr_at_100",
"value": 42.451,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 42.498000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 38.659,
"verified": false
},
{
"type": "mrr_at_5",
"value": 40.314,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 32.05,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.132,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.029,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 50.229,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.622,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 39.062000000000005,
"verified": false
},
{
"type": "precision_at_1",
"value": 32.05,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.767,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.269,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.164,
"verified": false
},
{
"type": "precision_at_3",
"value": 17.355999999999998,
"verified": false
},
{
"type": "precision_at_5",
"value": 12.474,
"verified": false
},
{
"type": "recall_at_1",
"value": 26.15,
"verified": false
},
{
"type": "recall_at_10",
"value": 55.205000000000005,
"verified": false
},
{
"type": "recall_at_100",
"value": 80.2,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.524,
"verified": false
},
{
"type": "recall_at_3",
"value": 39.322,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.761,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackProgrammersRetrieval",
"revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.741,
"verified": false
},
{
"type": "map_at_10",
"value": 33.51,
"verified": false
},
{
"type": "map_at_100",
"value": 34.882999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.995,
"verified": false
},
{
"type": "map_at_3",
"value": 30.514000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.085,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.653,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.059,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.05,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.107,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.445,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.849,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.653,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.186,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 45.301,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.547,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.103,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.239,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.653,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.295,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.219,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.159,
"verified": false
},
{
"type": "precision_at_3",
"value": 16.438,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.804,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.741,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.675,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.13799999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.127,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.033,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.793,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "map_at_10",
"value": 34.080666666666666,
"verified": false
},
{
"type": "map_at_100",
"value": 35.278749999999995,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.40183333333333,
"verified": false
},
{
"type": "map_at_3",
"value": 31.45316666666667,
"verified": false
},
{
"type": "map_at_5",
"value": 32.92716666666667,
"verified": false
},
{
"type": "mrr_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.077333333333335,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.936499999999995,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.000249999999994,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.7735,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.07683333333334,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 39.18300000000001,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.444750000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.90316666666667,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.69308333333333,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.80316666666666,
"verified": false
},
{
"type": "precision_at_1",
"value": 29.78783333333333,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.820749999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.1224166666666666,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.1525,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.936333333333335,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.282333333333334,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.281666666666663,
"verified": false
},
{
"type": "recall_at_10",
"value": 50.282,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.54558333333334,
"verified": false
},
{
"type": "recall_at_1000",
"value": 90.64241666666666,
"verified": false
},
{
"type": "recall_at_3",
"value": 37.800999999999995,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.223000000000006,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackStatsRetrieval",
"revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 23.452,
"verified": false
},
{
"type": "map_at_10",
"value": 30.231,
"verified": false
},
{
"type": "map_at_100",
"value": 31.227,
"verified": false
},
{
"type": "map_at_1000",
"value": 31.338,
"verified": false
},
{
"type": "map_at_3",
"value": 28.083,
"verified": false
},
{
"type": "map_at_5",
"value": 29.125,
"verified": false
},
{
"type": "mrr_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "mrr_at_10",
"value": 32.62,
"verified": false
},
{
"type": "mrr_at_100",
"value": 33.469,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 33.554,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.368,
"verified": false
},
{
"type": "mrr_at_5",
"value": 31.502999999999997,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 34.441,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 39.253,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 42.105,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 30.183,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 31.917,
"verified": false
},
{
"type": "precision_at_1",
"value": 25.613000000000003,
"verified": false
},
{
"type": "precision_at_10",
"value": 5.367999999999999,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.848,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.117,
"verified": false
},
{
"type": "precision_at_3",
"value": 12.73,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.773,
"verified": false
},
{
"type": "recall_at_1",
"value": 23.452,
"verified": false
},
{
"type": "recall_at_10",
"value": 45.021,
"verified": false
},
{
"type": "recall_at_100",
"value": 66.563,
"verified": false
},
{
"type": "recall_at_1000",
"value": 87.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 33.433,
"verified": false
},
{
"type": "recall_at_5",
"value": 37.637,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackTexRetrieval",
"revision": "46989137a86843e03a6195de44b09deda022eec7",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 16.11,
"verified": false
},
{
"type": "map_at_10",
"value": 22.832,
"verified": false
},
{
"type": "map_at_100",
"value": 23.829,
"verified": false
},
{
"type": "map_at_1000",
"value": 23.959,
"verified": false
},
{
"type": "map_at_3",
"value": 20.66,
"verified": false
},
{
"type": "map_at_5",
"value": 21.851000000000003,
"verified": false
},
{
"type": "mrr_at_1",
"value": 19.408,
"verified": false
},
{
"type": "mrr_at_10",
"value": 26.354,
"verified": false
},
{
"type": "mrr_at_100",
"value": 27.237,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 27.32,
"verified": false
},
{
"type": "mrr_at_3",
"value": 24.243,
"verified": false
},
{
"type": "mrr_at_5",
"value": 25.430000000000003,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 19.408,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 27.239,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.286,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 35.498000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 23.244,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 25.080999999999996,
"verified": false
},
{
"type": "precision_at_1",
"value": 19.408,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.917,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.874,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.133,
"verified": false
},
{
"type": "precision_at_3",
"value": 10.863,
"verified": false
},
{
"type": "precision_at_5",
"value": 7.887,
"verified": false
},
{
"type": "recall_at_1",
"value": 16.11,
"verified": false
},
{
"type": "recall_at_10",
"value": 37.075,
"verified": false
},
{
"type": "recall_at_100",
"value": 60.251999999999995,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.38600000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 25.901999999999997,
"verified": false
},
{
"type": "recall_at_5",
"value": 30.612,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackUnixRetrieval",
"revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 25.941,
"verified": false
},
{
"type": "map_at_10",
"value": 33.711999999999996,
"verified": false
},
{
"type": "map_at_100",
"value": 34.926,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.05,
"verified": false
},
{
"type": "map_at_3",
"value": 31.075000000000003,
"verified": false
},
{
"type": "map_at_5",
"value": 32.611000000000004,
"verified": false
},
{
"type": "mrr_at_1",
"value": 30.784,
"verified": false
},
{
"type": "mrr_at_10",
"value": 38.079,
"verified": false
},
{
"type": "mrr_at_100",
"value": 39.018,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 39.09,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.603,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.988,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 30.784,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.586,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.205,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.916,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 33.899,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.11,
"verified": false
},
{
"type": "precision_at_1",
"value": 30.784,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.409,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.034,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.13799999999999998,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.112,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.728,
"verified": false
},
{
"type": "recall_at_1",
"value": 25.941,
"verified": false
},
{
"type": "recall_at_10",
"value": 49.242999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 73.85000000000001,
"verified": false
},
{
"type": "recall_at_1000",
"value": 92.782,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.204,
"verified": false
},
{
"type": "recall_at_5",
"value": 41.908,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWebmastersRetrieval",
"revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 33.195,
"verified": false
},
{
"type": "map_at_100",
"value": 34.699999999999996,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.946,
"verified": false
},
{
"type": "map_at_3",
"value": 30.571,
"verified": false
},
{
"type": "map_at_5",
"value": 32,
"verified": false
},
{
"type": "mrr_at_1",
"value": 28.656,
"verified": false
},
{
"type": "mrr_at_10",
"value": 37.039,
"verified": false
},
{
"type": "mrr_at_100",
"value": 38.049,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 38.108,
"verified": false
},
{
"type": "mrr_at_3",
"value": 34.717,
"verified": false
},
{
"type": "mrr_at_5",
"value": 36.07,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 28.656,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 38.557,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 44.511,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.346,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 34.235,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.261,
"verified": false
},
{
"type": "precision_at_1",
"value": 28.656,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.312,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.451,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.242,
"verified": false
},
{
"type": "precision_at_3",
"value": 15.942,
"verified": false
},
{
"type": "precision_at_5",
"value": 11.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.401999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 48.791,
"verified": false
},
{
"type": "recall_at_100",
"value": 76.211,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.92,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.975,
"verified": false
},
{
"type": "recall_at_5",
"value": 42.01,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CQADupstackWordpressRetrieval",
"revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
"split": "test",
"type": "BeIR/cqadupstack"
},
"metrics": [
{
"type": "map_at_1",
"value": 19.07,
"verified": false
},
{
"type": "map_at_10",
"value": 26.609,
"verified": false
},
{
"type": "map_at_100",
"value": 27.625,
"verified": false
},
{
"type": "map_at_1000",
"value": 27.743,
"verified": false
},
{
"type": "map_at_3",
"value": 24.533,
"verified": false
},
{
"type": "map_at_5",
"value": 25.671,
"verified": false
},
{
"type": "mrr_at_1",
"value": 20.518,
"verified": false
},
{
"type": "mrr_at_10",
"value": 28.541,
"verified": false
},
{
"type": "mrr_at_100",
"value": 29.453000000000003,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 29.536,
"verified": false
},
{
"type": "mrr_at_3",
"value": 26.71,
"verified": false
},
{
"type": "mrr_at_5",
"value": 27.708,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 20.518,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 30.855,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 35.973,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.827,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 26.868,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 28.74,
"verified": false
},
{
"type": "precision_at_1",
"value": 20.518,
"verified": false
},
{
"type": "precision_at_10",
"value": 4.843,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.799,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 11.645,
"verified": false
},
{
"type": "precision_at_5",
"value": 8.133,
"verified": false
},
{
"type": "recall_at_1",
"value": 19.07,
"verified": false
},
{
"type": "recall_at_10",
"value": 41.925,
"verified": false
},
{
"type": "recall_at_100",
"value": 65.68,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.713,
"verified": false
},
{
"type": "recall_at_3",
"value": 31.251,
"verified": false
},
{
"type": "recall_at_5",
"value": 35.653,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ClimateFEVER",
"revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
"split": "test",
"type": "mteb/climate-fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 18.762,
"verified": false
},
{
"type": "map_at_10",
"value": 32.412,
"verified": false
},
{
"type": "map_at_100",
"value": 34.506,
"verified": false
},
{
"type": "map_at_1000",
"value": 34.678,
"verified": false
},
{
"type": "map_at_3",
"value": 27.594,
"verified": false
},
{
"type": "map_at_5",
"value": 30.128,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.345,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.443,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.05799999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.076,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.553,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.269,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 42.345,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 42.304,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 49.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 52.123,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.271,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 38.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.345,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.808,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.062,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.258,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.840000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 20.052,
"verified": false
},
{
"type": "recall_at_1",
"value": 18.762,
"verified": false
},
{
"type": "recall_at_10",
"value": 47.976,
"verified": false
},
{
"type": "recall_at_100",
"value": 71.86,
"verified": false
},
{
"type": "recall_at_1000",
"value": 86.61999999999999,
"verified": false
},
{
"type": "recall_at_3",
"value": 32.708999999999996,
"verified": false
},
{
"type": "recall_at_5",
"value": 39.151,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CmedqaRetrieval",
"revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"split": "dev",
"type": "C-MTEB/CmedqaRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.871,
"verified": false
},
{
"type": "map_at_10",
"value": 37.209,
"verified": false
},
{
"type": "map_at_100",
"value": 38.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.122,
"verified": false
},
{
"type": "map_at_3",
"value": 33.2,
"verified": false
},
{
"type": "map_at_5",
"value": 35.33,
"verified": false
},
{
"type": "mrr_at_1",
"value": 37.884,
"verified": false
},
{
"type": "mrr_at_10",
"value": 46.189,
"verified": false
},
{
"type": "mrr_at_100",
"value": 47.147,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 47.195,
"verified": false
},
{
"type": "mrr_at_3",
"value": 43.728,
"verified": false
},
{
"type": "mrr_at_5",
"value": 44.994,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 37.884,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 43.878,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 51.002,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 53.161,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 38.729,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.628,
"verified": false
},
{
"type": "precision_at_1",
"value": 37.884,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.75,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.558,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.183,
"verified": false
},
{
"type": "precision_at_3",
"value": 21.964,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.719,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.871,
"verified": false
},
{
"type": "recall_at_10",
"value": 54.615,
"verified": false
},
{
"type": "recall_at_100",
"value": 84.276,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.578,
"verified": false
},
{
"type": "recall_at_3",
"value": 38.936,
"verified": false
},
{
"type": "recall_at_5",
"value": 45.061,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Cmnli",
"revision": "41bc36f332156f7adc9e38f53777c959b2ae9766",
"split": "validation",
"type": "C-MTEB/CMNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.47948419710998,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "dot_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "dot_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "dot_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "dot_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "dot_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.47947997540409,
"verified": false
},
{
"type": "euclidean_f1",
"value": 77.88108108108108,
"verified": false
},
{
"type": "euclidean_precision",
"value": 72.43112809169516,
"verified": false
},
{
"type": "euclidean_recall",
"value": 84.21790974982464,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 75.40589296452195,
"verified": false
},
{
"type": "manhattan_ap",
"value": 83.74383956930585,
"verified": false
},
{
"type": "manhattan_f1",
"value": 77.0983342289092,
"verified": false
},
{
"type": "manhattan_precision",
"value": 71.34049323786795,
"verified": false
},
{
"type": "manhattan_recall",
"value": 83.86719663315408,
"verified": false
},
{
"type": "max_accuracy",
"value": 76.12748045700542,
"verified": false
},
{
"type": "max_ap",
"value": 84.4933237839786,
"verified": false
},
{
"type": "max_f1",
"value": 77.88108108108108,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB CovidRetrieval",
"revision": "1271c7809071a13532e05f25fb53511ffce77117",
"split": "dev",
"type": "C-MTEB/CovidRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 66.781,
"verified": false
},
{
"type": "map_at_10",
"value": 74.539,
"verified": false
},
{
"type": "map_at_100",
"value": 74.914,
"verified": false
},
{
"type": "map_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "map_at_3",
"value": 72.734,
"verified": false
},
{
"type": "map_at_5",
"value": 73.788,
"verified": false
},
{
"type": "mrr_at_1",
"value": 66.913,
"verified": false
},
{
"type": "mrr_at_10",
"value": 74.543,
"verified": false
},
{
"type": "mrr_at_100",
"value": 74.914,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 74.921,
"verified": false
},
{
"type": "mrr_at_3",
"value": 72.831,
"verified": false
},
{
"type": "mrr_at_5",
"value": 73.76899999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 67.018,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 78.34299999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 80.138,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 80.322,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 74.667,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 76.518,
"verified": false
},
{
"type": "precision_at_1",
"value": 67.018,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.115,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.996,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.101,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.906,
"verified": false
},
{
"type": "precision_at_5",
"value": 17.092,
"verified": false
},
{
"type": "recall_at_1",
"value": 66.781,
"verified": false
},
{
"type": "recall_at_10",
"value": 90.253,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.525,
"verified": false
},
{
"type": "recall_at_1000",
"value": 100,
"verified": false
},
{
"type": "recall_at_3",
"value": 80.05799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.615,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DBPedia",
"revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
"split": "test",
"type": "mteb/dbpedia"
},
"metrics": [
{
"type": "map_at_1",
"value": 9.685,
"verified": false
},
{
"type": "map_at_10",
"value": 21.65,
"verified": false
},
{
"type": "map_at_100",
"value": 30.952,
"verified": false
},
{
"type": "map_at_1000",
"value": 33.049,
"verified": false
},
{
"type": "map_at_3",
"value": 14.953,
"verified": false
},
{
"type": "map_at_5",
"value": 17.592,
"verified": false
},
{
"type": "mrr_at_1",
"value": 72,
"verified": false
},
{
"type": "mrr_at_10",
"value": 78.054,
"verified": false
},
{
"type": "mrr_at_100",
"value": 78.41900000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 78.425,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.5,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.28699999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 61.25000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 46.306,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 50.867,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 58.533,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 50.857,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 48.283,
"verified": false
},
{
"type": "precision_at_1",
"value": 72,
"verified": false
},
{
"type": "precision_at_10",
"value": 37.3,
"verified": false
},
{
"type": "precision_at_100",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.528,
"verified": false
},
{
"type": "precision_at_3",
"value": 53.583000000000006,
"verified": false
},
{
"type": "precision_at_5",
"value": 46.6,
"verified": false
},
{
"type": "recall_at_1",
"value": 9.685,
"verified": false
},
{
"type": "recall_at_10",
"value": 27.475,
"verified": false
},
{
"type": "recall_at_100",
"value": 56.825,
"verified": false
},
{
"type": "recall_at_1000",
"value": 81.792,
"verified": false
},
{
"type": "recall_at_3",
"value": 15.939,
"verified": false
},
{
"type": "recall_at_5",
"value": 19.853,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB DuRetrieval",
"revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced",
"split": "dev",
"type": "C-MTEB/DuRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.528,
"verified": false
},
{
"type": "map_at_10",
"value": 76.304,
"verified": false
},
{
"type": "map_at_100",
"value": 79.327,
"verified": false
},
{
"type": "map_at_1000",
"value": 79.373,
"verified": false
},
{
"type": "map_at_3",
"value": 52.035,
"verified": false
},
{
"type": "map_at_5",
"value": 66.074,
"verified": false
},
{
"type": "mrr_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 90.74,
"verified": false
},
{
"type": "mrr_at_100",
"value": 90.809,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 90.811,
"verified": false
},
{
"type": "mrr_at_3",
"value": 90.308,
"verified": false
},
{
"type": "mrr_at_5",
"value": 90.601,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 84.518,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 87.779,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.184,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 82.339,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 81.613,
"verified": false
},
{
"type": "precision_at_1",
"value": 86.05000000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 40.945,
"verified": false
},
{
"type": "precision_at_100",
"value": 4.787,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.4890000000000001,
"verified": false
},
{
"type": "precision_at_3",
"value": 74.117,
"verified": false
},
{
"type": "precision_at_5",
"value": 62.86000000000001,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.528,
"verified": false
},
{
"type": "recall_at_10",
"value": 86.78,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.198,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.227,
"verified": false
},
{
"type": "recall_at_3",
"value": 54.94799999999999,
"verified": false
},
{
"type": "recall_at_5",
"value": 72.053,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EcomRetrieval",
"revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9",
"split": "dev",
"type": "C-MTEB/EcomRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 52.1,
"verified": false
},
{
"type": "map_at_10",
"value": 62.502,
"verified": false
},
{
"type": "map_at_100",
"value": 63.026,
"verified": false
},
{
"type": "map_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "map_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "map_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "mrr_at_1",
"value": 52.1,
"verified": false
},
{
"type": "mrr_at_10",
"value": 62.502,
"verified": false
},
{
"type": "mrr_at_100",
"value": 63.026,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 63.04,
"verified": false
},
{
"type": "mrr_at_3",
"value": 59.782999999999994,
"verified": false
},
{
"type": "mrr_at_5",
"value": 61.443000000000005,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 52.1,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 67.75999999999999,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 70.072,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 70.441,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 62.28,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 65.25800000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 52.1,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.43,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.946,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.098,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.167,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.340000000000002,
"verified": false
},
{
"type": "recall_at_1",
"value": 52.1,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.3,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.6,
"verified": false
},
{
"type": "recall_at_1000",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_3",
"value": 69.5,
"verified": false
},
{
"type": "recall_at_5",
"value": 76.7,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB EmotionClassification",
"revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
"split": "test",
"type": "mteb/emotion"
},
"metrics": [
{
"type": "accuracy",
"value": 62.805000000000014,
"verified": false
},
{
"type": "f1",
"value": 56.401757250989384,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FEVER",
"revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
"split": "test",
"type": "mteb/fever"
},
"metrics": [
{
"type": "map_at_1",
"value": 83.734,
"verified": false
},
{
"type": "map_at_10",
"value": 90.089,
"verified": false
},
{
"type": "map_at_100",
"value": 90.274,
"verified": false
},
{
"type": "map_at_1000",
"value": 90.286,
"verified": false
},
{
"type": "map_at_3",
"value": 89.281,
"verified": false
},
{
"type": "map_at_5",
"value": 89.774,
"verified": false
},
{
"type": "mrr_at_1",
"value": 90.039,
"verified": false
},
{
"type": "mrr_at_10",
"value": 94.218,
"verified": false
},
{
"type": "mrr_at_100",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 94.24,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.979,
"verified": false
},
{
"type": "mrr_at_5",
"value": 94.137,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 90.039,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 92.597,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 93.147,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 93.325,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 91.65,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 92.137,
"verified": false
},
{
"type": "precision_at_1",
"value": 90.039,
"verified": false
},
{
"type": "precision_at_10",
"value": 10.809,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.133,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.116,
"verified": false
},
{
"type": "precision_at_3",
"value": 34.338,
"verified": false
},
{
"type": "precision_at_5",
"value": 21.089,
"verified": false
},
{
"type": "recall_at_1",
"value": 83.734,
"verified": false
},
{
"type": "recall_at_10",
"value": 96.161,
"verified": false
},
{
"type": "recall_at_100",
"value": 98.137,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.182,
"verified": false
},
{
"type": "recall_at_3",
"value": 93.551,
"verified": false
},
{
"type": "recall_at_5",
"value": 94.878,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB FiQA2018",
"revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
"split": "test",
"type": "mteb/fiqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 24.53,
"verified": false
},
{
"type": "map_at_10",
"value": 37.229,
"verified": false
},
{
"type": "map_at_100",
"value": 39.333,
"verified": false
},
{
"type": "map_at_1000",
"value": 39.491,
"verified": false
},
{
"type": "map_at_3",
"value": 32.177,
"verified": false
},
{
"type": "map_at_5",
"value": 35.077999999999996,
"verified": false
},
{
"type": "mrr_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "mrr_at_10",
"value": 53.952,
"verified": false
},
{
"type": "mrr_at_100",
"value": 54.727,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 54.761,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.569,
"verified": false
},
{
"type": "mrr_at_5",
"value": 52.973000000000006,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 45.297,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 52.516,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 55.16,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 40.569,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 42.49,
"verified": false
},
{
"type": "precision_at_1",
"value": 45.678999999999995,
"verified": false
},
{
"type": "precision_at_10",
"value": 12.269,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.971,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.244,
"verified": false
},
{
"type": "precision_at_3",
"value": 25.72,
"verified": false
},
{
"type": "precision_at_5",
"value": 19.66,
"verified": false
},
{
"type": "recall_at_1",
"value": 24.53,
"verified": false
},
{
"type": "recall_at_10",
"value": 51.983999999999995,
"verified": false
},
{
"type": "recall_at_100",
"value": 78.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 94.104,
"verified": false
},
{
"type": "recall_at_3",
"value": 36.45,
"verified": false
},
{
"type": "recall_at_5",
"value": 43.337,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB HotpotQA",
"revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
"split": "test",
"type": "mteb/hotpotqa"
},
"metrics": [
{
"type": "map_at_1",
"value": 41.519,
"verified": false
},
{
"type": "map_at_10",
"value": 64.705,
"verified": false
},
{
"type": "map_at_100",
"value": 65.554,
"verified": false
},
{
"type": "map_at_1000",
"value": 65.613,
"verified": false
},
{
"type": "map_at_3",
"value": 61.478,
"verified": false
},
{
"type": "map_at_5",
"value": 63.55800000000001,
"verified": false
},
{
"type": "mrr_at_1",
"value": 83.038,
"verified": false
},
{
"type": "mrr_at_10",
"value": 87.82900000000001,
"verified": false
},
{
"type": "mrr_at_100",
"value": 87.96000000000001,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 87.96300000000001,
"verified": false
},
{
"type": "mrr_at_3",
"value": 87.047,
"verified": false
},
{
"type": "mrr_at_5",
"value": 87.546,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 83.038,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 72.928,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 75.778,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 76.866,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 68.46600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 71.036,
"verified": false
},
{
"type": "precision_at_1",
"value": 83.038,
"verified": false
},
{
"type": "precision_at_10",
"value": 15.041,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.7260000000000002,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.187,
"verified": false
},
{
"type": "precision_at_3",
"value": 43.597,
"verified": false
},
{
"type": "precision_at_5",
"value": 28.188999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 41.519,
"verified": false
},
{
"type": "recall_at_10",
"value": 75.20599999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 86.3,
"verified": false
},
{
"type": "recall_at_1000",
"value": 93.437,
"verified": false
},
{
"type": "recall_at_3",
"value": 65.39500000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.473,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB IFlyTek",
"revision": "421605374b29664c5fc098418fe20ada9bd55f8a",
"split": "validation",
"type": "C-MTEB/IFlyTek-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 52.04309349749903,
"verified": false
},
{
"type": "f1",
"value": 39.91893257315586,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ImdbClassification",
"revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
"split": "test",
"type": "mteb/imdb"
},
"metrics": [
{
"type": "accuracy",
"value": 96.0428,
"verified": false
},
{
"type": "ap",
"value": 94.48278082595031,
"verified": false
},
{
"type": "f1",
"value": 96.0409595432081,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB JDReview",
"revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b",
"split": "test",
"type": "C-MTEB/JDReview-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 85.60975609756099,
"verified": false
},
{
"type": "ap",
"value": 54.30148799475452,
"verified": false
},
{
"type": "f1",
"value": 80.55899583002706,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB LCQMC",
"revision": "17f9b096f80380fce5ed12a9be8be7784b337daf",
"split": "test",
"type": "C-MTEB/LCQMC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 66.44418108776416,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.79912770347306,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 71.11194894579198,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.79912104971427,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.96800061808604,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.63525186107175,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoReranking",
"revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6",
"split": "dev",
"type": "C-MTEB/Mmarco-reranking"
},
"metrics": [
{
"type": "map",
"value": 27.9616280919871,
"verified": false
},
{
"type": "mrr",
"value": 26.544047619047618,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MMarcoRetrieval",
"revision": "539bbde593d947e2a124ba72651aafc09eb33fc2",
"split": "dev",
"type": "C-MTEB/MMarcoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "map_at_10",
"value": 77.187,
"verified": false
},
{
"type": "map_at_100",
"value": 77.496,
"verified": false
},
{
"type": "map_at_1000",
"value": 77.503,
"verified": false
},
{
"type": "map_at_3",
"value": 75.405,
"verified": false
},
{
"type": "map_at_5",
"value": 76.539,
"verified": false
},
{
"type": "mrr_at_1",
"value": 70.616,
"verified": false
},
{
"type": "mrr_at_10",
"value": 77.703,
"verified": false
},
{
"type": "mrr_at_100",
"value": 77.97699999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 77.984,
"verified": false
},
{
"type": "mrr_at_3",
"value": 76.139,
"verified": false
},
{
"type": "mrr_at_5",
"value": 77.125,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 70.616,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 80.741,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 82.123,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 82.32300000000001,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 77.35600000000001,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 79.274,
"verified": false
},
{
"type": "precision_at_1",
"value": 70.616,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.696,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.038,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.106,
"verified": false
},
{
"type": "precision_at_3",
"value": 29.026000000000003,
"verified": false
},
{
"type": "precision_at_5",
"value": 18.433,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.32300000000001,
"verified": false
},
{
"type": "recall_at_10",
"value": 91.186,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.439,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.004,
"verified": false
},
{
"type": "recall_at_3",
"value": 82.218,
"verified": false
},
{
"type": "recall_at_5",
"value": 86.797,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MSMARCO",
"revision": "c5a29a104738b98a9e76336939199e264163d4a0",
"split": "dev",
"type": "mteb/msmarco"
},
"metrics": [
{
"type": "map_at_1",
"value": 21.496,
"verified": false
},
{
"type": "map_at_10",
"value": 33.82,
"verified": false
},
{
"type": "map_at_100",
"value": 35.013,
"verified": false
},
{
"type": "map_at_1000",
"value": 35.063,
"verified": false
},
{
"type": "map_at_3",
"value": 29.911,
"verified": false
},
{
"type": "map_at_5",
"value": 32.086,
"verified": false
},
{
"type": "mrr_at_1",
"value": 22.092,
"verified": false
},
{
"type": "mrr_at_10",
"value": 34.404,
"verified": false
},
{
"type": "mrr_at_100",
"value": 35.534,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 35.577999999999996,
"verified": false
},
{
"type": "mrr_at_3",
"value": 30.544,
"verified": false
},
{
"type": "mrr_at_5",
"value": 32.711,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 22.092,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 40.877,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 46.619,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 47.823,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 32.861000000000004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 36.769,
"verified": false
},
{
"type": "precision_at_1",
"value": 22.092,
"verified": false
},
{
"type": "precision_at_10",
"value": 6.54,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.943,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.105,
"verified": false
},
{
"type": "precision_at_3",
"value": 14.069,
"verified": false
},
{
"type": "precision_at_5",
"value": 10.424,
"verified": false
},
{
"type": "recall_at_1",
"value": 21.496,
"verified": false
},
{
"type": "recall_at_10",
"value": 62.67,
"verified": false
},
{
"type": "recall_at_100",
"value": 89.24499999999999,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.312,
"verified": false
},
{
"type": "recall_at_3",
"value": 40.796,
"verified": false
},
{
"type": "recall_at_5",
"value": 50.21600000000001,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPDomainClassification (en)",
"revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
"split": "test",
"type": "mteb/mtop_domain"
},
"metrics": [
{
"type": "accuracy",
"value": 95.74555403556772,
"verified": false
},
{
"type": "f1",
"value": 95.61381879323092,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MTOPIntentClassification (en)",
"revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
"split": "test",
"type": "mteb/mtop_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 85.82763337893297,
"verified": false
},
{
"type": "f1",
"value": 63.17139719465236,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveIntentClassification (en)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 78.51714862138535,
"verified": false
},
{
"type": "f1",
"value": 76.3995118440293,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveIntentClassification (zh-CN)",
"revision": "31efe3c427b0bae9c22cbb560b8f15491cc6bed7",
"split": "test",
"type": "mteb/amazon_massive_intent"
},
"metrics": [
{
"type": "accuracy",
"value": 74.78143913920646,
"verified": false
},
{
"type": "f1",
"value": 72.6141122227626,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB MassiveScenarioClassification (en)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 80.03698722259583,
"verified": false
},
{
"type": "f1",
"value": 79.36511484240766,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "zh-CN",
"name": "MTEB MassiveScenarioClassification (zh-CN)",
"revision": "7d571f92784cd94a019292a1f45445077d0ef634",
"split": "test",
"type": "mteb/amazon_massive_scenario"
},
"metrics": [
{
"type": "accuracy",
"value": 76.98722259583053,
"verified": false
},
{
"type": "f1",
"value": 76.5974920207624,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedicalRetrieval",
"revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6",
"split": "dev",
"type": "C-MTEB/MedicalRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 51.8,
"verified": false
},
{
"type": "map_at_10",
"value": 57.938,
"verified": false
},
{
"type": "map_at_100",
"value": 58.494,
"verified": false
},
{
"type": "map_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "map_at_3",
"value": 56.617,
"verified": false
},
{
"type": "map_at_5",
"value": 57.302,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51.8,
"verified": false
},
{
"type": "mrr_at_10",
"value": 57.938,
"verified": false
},
{
"type": "mrr_at_100",
"value": 58.494,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 58.541,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.617,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.302,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51.8,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.891,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.897000000000006,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 65.231,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 58.108,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 59.343,
"verified": false
},
{
"type": "precision_at_1",
"value": 51.8,
"verified": false
},
{
"type": "precision_at_10",
"value": 7.02,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.8500000000000001,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.096,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.8,
"verified": false
},
{
"type": "precision_at_5",
"value": 13.08,
"verified": false
},
{
"type": "recall_at_1",
"value": 51.8,
"verified": false
},
{
"type": "recall_at_10",
"value": 70.19999999999999,
"verified": false
},
{
"type": "recall_at_100",
"value": 85,
"verified": false
},
{
"type": "recall_at_1000",
"value": 95.7,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.4,
"verified": false
},
{
"type": "recall_at_5",
"value": 65.4,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringP2P",
"revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
"split": "test",
"type": "mteb/medrxiv-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 38.68901889835701,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MedrxivClusteringS2S",
"revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
"split": "test",
"type": "mteb/medrxiv-clustering-s2s"
},
"metrics": [
{
"type": "v_measure",
"value": 38.0740589898848,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MindSmallReranking",
"revision": "3bdac13927fdc888b903db93b2ffdbd90b295a69",
"split": "test",
"type": "mteb/mind_small"
},
"metrics": [
{
"type": "map",
"value": 33.41312482460189,
"verified": false
},
{
"type": "mrr",
"value": 34.713530863302495,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB MultilingualSentiment",
"revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a",
"split": "validation",
"type": "C-MTEB/MultilingualSentiment-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 80.39333333333335,
"verified": false
},
{
"type": "f1",
"value": 80.42683132366277,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NFCorpus",
"revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
"split": "test",
"type": "mteb/nfcorpus"
},
"metrics": [
{
"type": "map_at_1",
"value": 6.232,
"verified": false
},
{
"type": "map_at_10",
"value": 13.442000000000002,
"verified": false
},
{
"type": "map_at_100",
"value": 17.443,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.1,
"verified": false
},
{
"type": "map_at_3",
"value": 9.794,
"verified": false
},
{
"type": "map_at_5",
"value": 11.375,
"verified": false
},
{
"type": "mrr_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.628,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.077,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.119,
"verified": false
},
{
"type": "mrr_at_3",
"value": 56.914,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.921,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 48.762,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 37.203,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 34.556,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 43.601,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 43.004,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 40.181,
"verified": false
},
{
"type": "precision_at_1",
"value": 50.15500000000001,
"verified": false
},
{
"type": "precision_at_10",
"value": 27.276,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.981,
"verified": false
},
{
"type": "precision_at_1000",
"value": 2.228,
"verified": false
},
{
"type": "precision_at_3",
"value": 39.628,
"verified": false
},
{
"type": "precision_at_5",
"value": 33.808,
"verified": false
},
{
"type": "recall_at_1",
"value": 6.232,
"verified": false
},
{
"type": "recall_at_10",
"value": 18.137,
"verified": false
},
{
"type": "recall_at_100",
"value": 36.101,
"verified": false
},
{
"type": "recall_at_1000",
"value": 68.733,
"verified": false
},
{
"type": "recall_at_3",
"value": 10.978,
"verified": false
},
{
"type": "recall_at_5",
"value": 13.718,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB NQ",
"revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
"split": "test",
"type": "mteb/nq"
},
"metrics": [
{
"type": "map_at_1",
"value": 35.545,
"verified": false
},
{
"type": "map_at_10",
"value": 52.083,
"verified": false
},
{
"type": "map_at_100",
"value": 52.954,
"verified": false
},
{
"type": "map_at_1000",
"value": 52.96999999999999,
"verified": false
},
{
"type": "map_at_3",
"value": 47.508,
"verified": false
},
{
"type": "map_at_5",
"value": 50.265,
"verified": false
},
{
"type": "mrr_at_1",
"value": 40.122,
"verified": false
},
{
"type": "mrr_at_10",
"value": 54.567,
"verified": false
},
{
"type": "mrr_at_100",
"value": 55.19199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 55.204,
"verified": false
},
{
"type": "mrr_at_3",
"value": 51.043000000000006,
"verified": false
},
{
"type": "mrr_at_5",
"value": 53.233,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 40.122,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 60.012,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 63.562,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 63.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 51.681,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 56.154,
"verified": false
},
{
"type": "precision_at_1",
"value": 40.122,
"verified": false
},
{
"type": "precision_at_10",
"value": 9.774,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.176,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.121,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.426,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.686,
"verified": false
},
{
"type": "recall_at_1",
"value": 35.545,
"verified": false
},
{
"type": "recall_at_10",
"value": 81.557,
"verified": false
},
{
"type": "recall_at_100",
"value": 96.729,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.541,
"verified": false
},
{
"type": "recall_at_3",
"value": 60.185,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.411,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Ocnli",
"revision": "66e76a618a34d6d565d5538088562851e6daa7ec",
"split": "validation",
"type": "C-MTEB/OCNLI"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "dot_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "dot_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "dot_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "dot_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "dot_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "euclidean_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "euclidean_f1",
"value": 72.77277277277277,
"verified": false
},
{
"type": "euclidean_precision",
"value": 69.17221693625119,
"verified": false
},
{
"type": "euclidean_recall",
"value": 76.76874340021119,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 69.89713048186248,
"verified": false
},
{
"type": "manhattan_ap",
"value": 74.25943370061067,
"verified": false
},
{
"type": "manhattan_f1",
"value": 72.17268887846082,
"verified": false
},
{
"type": "manhattan_precision",
"value": 64.94932432432432,
"verified": false
},
{
"type": "manhattan_recall",
"value": 81.20380147835269,
"verified": false
},
{
"type": "max_accuracy",
"value": 70.7634001082837,
"verified": false
},
{
"type": "max_ap",
"value": 74.97527385556558,
"verified": false
},
{
"type": "max_f1",
"value": 72.77277277277277,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB OnlineShopping",
"revision": "e610f2ebd179a8fda30ae534c3878750a96db120",
"split": "test",
"type": "C-MTEB/OnlineShopping-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 92.92000000000002,
"verified": false
},
{
"type": "ap",
"value": 91.984756251062,
"verified": false
},
{
"type": "f1",
"value": 92.918414705419,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB PAWSX",
"revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1",
"split": "test",
"type": "C-MTEB/PAWSX"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 41.23764415526825,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 46.434144530918566,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 46.872669471694664,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 46.39678126910133,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 46.55877754642116,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QBQTC",
"revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7",
"split": "test",
"type": "C-MTEB/QBQTC"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 28.77503601696299,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 31.81809555732561,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 29.811479220397125,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 31.817046821577673,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 29.901628633314218,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 31.991472038092084,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB QuoraRetrieval",
"revision": "None",
"split": "test",
"type": "mteb/quora"
},
"metrics": [
{
"type": "map_at_1",
"value": 68.908,
"verified": false
},
{
"type": "map_at_10",
"value": 83.19,
"verified": false
},
{
"type": "map_at_100",
"value": 83.842,
"verified": false
},
{
"type": "map_at_1000",
"value": 83.858,
"verified": false
},
{
"type": "map_at_3",
"value": 80.167,
"verified": false
},
{
"type": "map_at_5",
"value": 82.053,
"verified": false
},
{
"type": "mrr_at_1",
"value": 79.46,
"verified": false
},
{
"type": "mrr_at_10",
"value": 86.256,
"verified": false
},
{
"type": "mrr_at_100",
"value": 86.37,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 86.371,
"verified": false
},
{
"type": "mrr_at_3",
"value": 85.177,
"verified": false
},
{
"type": "mrr_at_5",
"value": 85.908,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 79.5,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 87.244,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 88.532,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 88.626,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 84.161,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 85.835,
"verified": false
},
{
"type": "precision_at_1",
"value": 79.5,
"verified": false
},
{
"type": "precision_at_10",
"value": 13.339,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.53,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.157,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.97,
"verified": false
},
{
"type": "precision_at_5",
"value": 24.384,
"verified": false
},
{
"type": "recall_at_1",
"value": 68.908,
"verified": false
},
{
"type": "recall_at_10",
"value": 95.179,
"verified": false
},
{
"type": "recall_at_100",
"value": 99.579,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.964,
"verified": false
},
{
"type": "recall_at_3",
"value": 86.424,
"verified": false
},
{
"type": "recall_at_5",
"value": 91.065,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClustering",
"revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
"split": "test",
"type": "mteb/reddit-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 65.17897847862794,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB RedditClusteringP2P",
"revision": "282350215ef01743dc01b456c7f5241fa8937f16",
"split": "test",
"type": "mteb/reddit-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 66.22194961632586,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SCIDOCS",
"revision": "None",
"split": "test",
"type": "mteb/scidocs"
},
"metrics": [
{
"type": "map_at_1",
"value": 5.668,
"verified": false
},
{
"type": "map_at_10",
"value": 13.921,
"verified": false
},
{
"type": "map_at_100",
"value": 16.391,
"verified": false
},
{
"type": "map_at_1000",
"value": 16.749,
"verified": false
},
{
"type": "map_at_3",
"value": 10.002,
"verified": false
},
{
"type": "map_at_5",
"value": 11.974,
"verified": false
},
{
"type": "mrr_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "mrr_at_10",
"value": 39.290000000000006,
"verified": false
},
{
"type": "mrr_at_100",
"value": 40.313,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 40.355999999999995,
"verified": false
},
{
"type": "mrr_at_3",
"value": 35.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 37.742,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 23.172,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 32.307,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 38.048,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 22.043,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 19.287000000000003,
"verified": false
},
{
"type": "precision_at_1",
"value": 27.800000000000004,
"verified": false
},
{
"type": "precision_at_10",
"value": 11.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 2.526,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.39,
"verified": false
},
{
"type": "precision_at_3",
"value": 20.433,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.84,
"verified": false
},
{
"type": "recall_at_1",
"value": 5.668,
"verified": false
},
{
"type": "recall_at_10",
"value": 24.22,
"verified": false
},
{
"type": "recall_at_100",
"value": 51.217,
"verified": false
},
{
"type": "recall_at_1000",
"value": 79.10000000000001,
"verified": false
},
{
"type": "recall_at_3",
"value": 12.443,
"verified": false
},
{
"type": "recall_at_5",
"value": 17.068,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SICK-R",
"revision": "a6ea5a8cab320b040a23452cc28066d9beae2cee",
"split": "test",
"type": "mteb/sickr-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 82.83535239748218,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 73.98553311584509,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 79.57336200069007,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 73.98553926018461,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 79.02277757114132,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 73.52350678760683,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS12",
"revision": "a0d554a64d88156834ff5ae9920b964011b16384",
"split": "test",
"type": "mteb/sts12-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 81.99055838690317,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 72.05290668592296,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.7130610313565,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 72.0529066787229,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 82.09213883730894,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 72.5171577483134,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS13",
"revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
"split": "test",
"type": "mteb/sts13-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.4685161191763,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 84.4847436140129,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.05016757016948,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 84.48474353891532,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.83064062713048,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 84.30431591842805,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS14",
"revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
"split": "test",
"type": "mteb/sts14-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 83.00171021092486,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.49758593915315,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 77.91329577609622,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.23255996803785,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 77.80027024941825,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS15",
"revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
"split": "test",
"type": "mteb/sts15-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 86.62608607472492,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 87.62293916855751,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 87.04313886714989,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 87.62293907119869,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 86.97266321040769,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 87.61807042381702,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STS16",
"revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
"split": "test",
"type": "mteb/sts16-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 80.8012095789289,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 81.2267973811213,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 81.91868918081325,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 81.0173457901168,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 81.79743115887055,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en-en",
"name": "MTEB STS17 (en-en)",
"revision": "af5e6fb845001ecf41f4c1e033ce921939a2a68d",
"split": "test",
"type": "mteb/sts17-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 88.39698537303725,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 88.78863351718252,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 88.78668529808967,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 88.41678215762478,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 88.3827998418763,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "en",
"name": "MTEB STS22 (en)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 68.49024974161408,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 70.48882819806336,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 69.19917146180619,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 70.86827961779932,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 69.38456983992613,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "zh",
"name": "MTEB STS22 (zh)",
"revision": "eea2b4fe26a775864c896887d910b76a8098ad3f",
"split": "test",
"type": "mteb/sts22-crosslingual-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 67.41628669863584,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 67.67834985311778,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 67.87238206703478,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 68.23423896742973,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 68.27069260687092,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSB",
"revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0",
"split": "test",
"type": "C-MTEB/STSB"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 77.31628954400037,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 76.83296022489624,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 76.69680425261211,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 76.83287843321102,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 76.65603163327958,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 76.80803503360451,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB STSBenchmark",
"revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831",
"split": "test",
"type": "mteb/stsbenchmark-sts"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 84.31376078795105,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "euclidean_pearson",
"value": 84.06630133719332,
"verified": false
},
{
"type": "euclidean_spearman",
"value": 83.3985199217591,
"verified": false
},
{
"type": "manhattan_pearson",
"value": 83.7896654474364,
"verified": false
},
{
"type": "manhattan_spearman",
"value": 83.1885039212299,
"verified": false
}
],
"task": {
"type": "STS"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciDocsRR",
"revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab",
"split": "test",
"type": "mteb/scidocs-reranking"
},
"metrics": [
{
"type": "map",
"value": 85.83161002188668,
"verified": false
},
{
"type": "mrr",
"value": 96.19253114351152,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SciFact",
"revision": "0228b52cf27578f30900b9e5271d331663a030d7",
"split": "test",
"type": "mteb/scifact"
},
"metrics": [
{
"type": "map_at_1",
"value": 48.133,
"verified": false
},
{
"type": "map_at_10",
"value": 58.541,
"verified": false
},
{
"type": "map_at_100",
"value": 59.34,
"verified": false
},
{
"type": "map_at_1000",
"value": 59.367999999999995,
"verified": false
},
{
"type": "map_at_3",
"value": 55.191,
"verified": false
},
{
"type": "map_at_5",
"value": 57.084,
"verified": false
},
{
"type": "mrr_at_1",
"value": 51,
"verified": false
},
{
"type": "mrr_at_10",
"value": 59.858,
"verified": false
},
{
"type": "mrr_at_100",
"value": 60.474,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 60.501000000000005,
"verified": false
},
{
"type": "mrr_at_3",
"value": 57.111,
"verified": false
},
{
"type": "mrr_at_5",
"value": 58.694,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 51,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 63.817,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 67.229,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 67.94,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 57.896,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 60.785999999999994,
"verified": false
},
{
"type": "precision_at_1",
"value": 51,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.933,
"verified": false
},
{
"type": "precision_at_100",
"value": 1.0699999999999998,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.113,
"verified": false
},
{
"type": "precision_at_3",
"value": 23.111,
"verified": false
},
{
"type": "precision_at_5",
"value": 15.733,
"verified": false
},
{
"type": "recall_at_1",
"value": 48.133,
"verified": false
},
{
"type": "recall_at_10",
"value": 78.922,
"verified": false
},
{
"type": "recall_at_100",
"value": 94.167,
"verified": false
},
{
"type": "recall_at_1000",
"value": 99.667,
"verified": false
},
{
"type": "recall_at_3",
"value": 62.806,
"verified": false
},
{
"type": "recall_at_5",
"value": 70.078,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SprintDuplicateQuestions",
"revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46",
"split": "test",
"type": "mteb/sprintduplicatequestions-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 97.725578864934,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 93.8,
"verified": false
},
{
"type": "dot_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "dot_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "dot_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "dot_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "dot_recall",
"value": 93.8,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 99.88415841584158,
"verified": false
},
{
"type": "euclidean_ap",
"value": 97.72557439066108,
"verified": false
},
{
"type": "euclidean_f1",
"value": 94.1294530858003,
"verified": false
},
{
"type": "euclidean_precision",
"value": 94.4612286002014,
"verified": false
},
{
"type": "euclidean_recall",
"value": 93.8,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "manhattan_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "manhattan_f1",
"value": 94.1825476429288,
"verified": false
},
{
"type": "manhattan_precision",
"value": 94.46680080482898,
"verified": false
},
{
"type": "manhattan_recall",
"value": 93.9,
"verified": false
},
{
"type": "max_accuracy",
"value": 99.88514851485148,
"verified": false
},
{
"type": "max_ap",
"value": 97.7332433405196,
"verified": false
},
{
"type": "max_f1",
"value": 94.1825476429288,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClustering",
"revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259",
"split": "test",
"type": "mteb/stackexchange-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 72.8168026381278,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackExchangeClusteringP2P",
"revision": "815ca46b2622cec33ccafc3735d572c266efdb44",
"split": "test",
"type": "mteb/stackexchange-clustering-p2p"
},
"metrics": [
{
"type": "v_measure",
"value": 44.30948635130784,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB StackOverflowDupQuestions",
"revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69",
"split": "test",
"type": "mteb/stackoverflowdupquestions-reranking"
},
"metrics": [
{
"type": "map",
"value": 54.11268548719803,
"verified": false
},
{
"type": "mrr",
"value": 55.08079747050335,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB SummEval",
"revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c",
"split": "test",
"type": "mteb/summeval"
},
"metrics": [
{
"type": "cos_sim_pearson",
"value": 30.82885852096243,
"verified": false
},
{
"type": "cos_sim_spearman",
"value": 30.800770979226076,
"verified": false
},
{
"type": "dot_pearson",
"value": 30.82885608827704,
"verified": false
},
{
"type": "dot_spearman",
"value": 30.800770979226076,
"verified": false
}
],
"task": {
"type": "Summarization"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Reranking",
"revision": "76631901a18387f85eaa53e5450019b87ad58ef9",
"split": "dev",
"type": "C-MTEB/T2Reranking"
},
"metrics": [
{
"type": "map",
"value": 66.73038448968596,
"verified": false
},
{
"type": "mrr",
"value": 77.26510193334836,
"verified": false
}
],
"task": {
"type": "Reranking"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB T2Retrieval",
"revision": "8731a845f1bf500a4f111cf1070785c793d10e64",
"split": "dev",
"type": "C-MTEB/T2Retrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 28.157,
"verified": false
},
{
"type": "map_at_10",
"value": 79.00399999999999,
"verified": false
},
{
"type": "map_at_100",
"value": 82.51899999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 82.577,
"verified": false
},
{
"type": "map_at_3",
"value": 55.614,
"verified": false
},
{
"type": "map_at_5",
"value": 68.292,
"verified": false
},
{
"type": "mrr_at_1",
"value": 91.167,
"verified": false
},
{
"type": "mrr_at_10",
"value": 93.391,
"verified": false
},
{
"type": "mrr_at_100",
"value": 93.467,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 93.47,
"verified": false
},
{
"type": "mrr_at_3",
"value": 93.001,
"verified": false
},
{
"type": "mrr_at_5",
"value": 93.254,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 91.167,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 86.155,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 89.425,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 89.983,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 87.516,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 86.148,
"verified": false
},
{
"type": "precision_at_1",
"value": 91.167,
"verified": false
},
{
"type": "precision_at_10",
"value": 42.697,
"verified": false
},
{
"type": "precision_at_100",
"value": 5.032,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.516,
"verified": false
},
{
"type": "precision_at_3",
"value": 76.45100000000001,
"verified": false
},
{
"type": "precision_at_5",
"value": 64.051,
"verified": false
},
{
"type": "recall_at_1",
"value": 28.157,
"verified": false
},
{
"type": "recall_at_10",
"value": 84.974,
"verified": false
},
{
"type": "recall_at_100",
"value": 95.759,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.583,
"verified": false
},
{
"type": "recall_at_3",
"value": 57.102,
"verified": false
},
{
"type": "recall_at_5",
"value": 71.383,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TNews",
"revision": "317f262bf1e6126357bbe89e875451e4b0938fe4",
"split": "validation",
"type": "C-MTEB/TNews-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 55.031,
"verified": false
},
{
"type": "f1",
"value": 53.07992810732314,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TRECCOVID",
"revision": "None",
"split": "test",
"type": "mteb/trec-covid"
},
"metrics": [
{
"type": "map_at_1",
"value": 0.204,
"verified": false
},
{
"type": "map_at_10",
"value": 1.27,
"verified": false
},
{
"type": "map_at_100",
"value": 7.993,
"verified": false
},
{
"type": "map_at_1000",
"value": 20.934,
"verified": false
},
{
"type": "map_at_3",
"value": 0.469,
"verified": false
},
{
"type": "map_at_5",
"value": 0.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 76,
"verified": false
},
{
"type": "mrr_at_10",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_100",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 84.967,
"verified": false
},
{
"type": "mrr_at_3",
"value": 83.667,
"verified": false
},
{
"type": "mrr_at_5",
"value": 84.967,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 69,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 59.243,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 48.784,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 46.966,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 64.14,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 61.60600000000001,
"verified": false
},
{
"type": "precision_at_1",
"value": 76,
"verified": false
},
{
"type": "precision_at_10",
"value": 62.6,
"verified": false
},
{
"type": "precision_at_100",
"value": 50.18,
"verified": false
},
{
"type": "precision_at_1000",
"value": 21.026,
"verified": false
},
{
"type": "precision_at_3",
"value": 68.667,
"verified": false
},
{
"type": "precision_at_5",
"value": 66,
"verified": false
},
{
"type": "recall_at_1",
"value": 0.204,
"verified": false
},
{
"type": "recall_at_10",
"value": 1.582,
"verified": false
},
{
"type": "recall_at_100",
"value": 11.988,
"verified": false
},
{
"type": "recall_at_1000",
"value": 44.994,
"verified": false
},
{
"type": "recall_at_3",
"value": 0.515,
"verified": false
},
{
"type": "recall_at_5",
"value": 0.844,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringP2P",
"revision": "5798586b105c0434e4f0fe5e767abe619442cf93",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringP2P"
},
"metrics": [
{
"type": "v_measure",
"value": 72.80915114296552,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ThuNewsClusteringS2S",
"revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d",
"split": "test",
"type": "C-MTEB/ThuNewsClusteringS2S"
},
"metrics": [
{
"type": "v_measure",
"value": 70.86374654127641,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Touche2020",
"revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f",
"split": "test",
"type": "mteb/touche2020"
},
"metrics": [
{
"type": "map_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "map_at_10",
"value": 11.566,
"verified": false
},
{
"type": "map_at_100",
"value": 17.645,
"verified": false
},
{
"type": "map_at_1000",
"value": 19.206,
"verified": false
},
{
"type": "map_at_3",
"value": 6.986000000000001,
"verified": false
},
{
"type": "map_at_5",
"value": 8.716,
"verified": false
},
{
"type": "mrr_at_1",
"value": 42.857,
"verified": false
},
{
"type": "mrr_at_10",
"value": 58.287,
"verified": false
},
{
"type": "mrr_at_100",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 59.111,
"verified": false
},
{
"type": "mrr_at_3",
"value": 55.102,
"verified": false
},
{
"type": "mrr_at_5",
"value": 57.449,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 39.796,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 29.059,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 40.629,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 51.446000000000005,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 36.254999999999995,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 32.216,
"verified": false
},
{
"type": "precision_at_1",
"value": 42.857,
"verified": false
},
{
"type": "precision_at_10",
"value": 23.469,
"verified": false
},
{
"type": "precision_at_100",
"value": 8.041,
"verified": false
},
{
"type": "precision_at_1000",
"value": 1.551,
"verified": false
},
{
"type": "precision_at_3",
"value": 36.735,
"verified": false
},
{
"type": "precision_at_5",
"value": 30.203999999999997,
"verified": false
},
{
"type": "recall_at_1",
"value": 3.3009999999999997,
"verified": false
},
{
"type": "recall_at_10",
"value": 17.267,
"verified": false
},
{
"type": "recall_at_100",
"value": 49.36,
"verified": false
},
{
"type": "recall_at_1000",
"value": 83.673,
"verified": false
},
{
"type": "recall_at_3",
"value": 8.049000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 11.38,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB ToxicConversationsClassification",
"revision": "d7c0de2777da35d6aae2200a62c6e0e5af397c4c",
"split": "test",
"type": "mteb/toxic_conversations_50k"
},
"metrics": [
{
"type": "accuracy",
"value": 88.7576,
"verified": false
},
{
"type": "ap",
"value": 35.52110634325751,
"verified": false
},
{
"type": "f1",
"value": 74.14476947482417,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TweetSentimentExtractionClassification",
"revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a",
"split": "test",
"type": "mteb/tweet_sentiment_extraction"
},
"metrics": [
{
"type": "accuracy",
"value": 73.52009054895304,
"verified": false
},
{
"type": "f1",
"value": 73.81407409876577,
"verified": false
}
],
"task": {
"type": "Classification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwentyNewsgroupsClustering",
"revision": "6125ec4e24fa026cec8a478383ee943acfbd5449",
"split": "test",
"type": "mteb/twentynewsgroups-clustering"
},
"metrics": [
{
"type": "v_measure",
"value": 54.35358706465052,
"verified": false
}
],
"task": {
"type": "Clustering"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterSemEval2015",
"revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1",
"split": "test",
"type": "mteb/twittersemeval2015-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 65.824087818991,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "dot_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "dot_ap",
"value": 65.82406256999921,
"verified": false
},
{
"type": "dot_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "dot_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "dot_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "euclidean_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "euclidean_f1",
"value": 61.952620244077536,
"verified": false
},
{
"type": "euclidean_precision",
"value": 56.676882661996494,
"verified": false
},
{
"type": "euclidean_recall",
"value": 68.311345646438,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 83.4296954163438,
"verified": false
},
{
"type": "manhattan_ap",
"value": 65.20662449614932,
"verified": false
},
{
"type": "manhattan_f1",
"value": 61.352885525070946,
"verified": false
},
{
"type": "manhattan_precision",
"value": 55.59365623660523,
"verified": false
},
{
"type": "manhattan_recall",
"value": 68.44327176781002,
"verified": false
},
{
"type": "max_accuracy",
"value": 83.65619598259522,
"verified": false
},
{
"type": "max_ap",
"value": 65.82409143427542,
"verified": false
},
{
"type": "max_f1",
"value": 61.952620244077536,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB TwitterURLCorpus",
"revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf",
"split": "test",
"type": "mteb/twitterurlcorpus-pairclassification"
},
"metrics": [
{
"type": "cos_sim_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "cos_sim_ap",
"value": 84.04753852793387,
"verified": false
},
{
"type": "cos_sim_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "cos_sim_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "cos_sim_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "dot_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "dot_ap",
"value": 84.04753668117337,
"verified": false
},
{
"type": "dot_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "dot_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "dot_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "euclidean_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "euclidean_ap",
"value": 84.04754553468206,
"verified": false
},
{
"type": "euclidean_f1",
"value": 76.27737226277372,
"verified": false
},
{
"type": "euclidean_precision",
"value": 73.86757068667052,
"verified": false
},
{
"type": "euclidean_recall",
"value": 78.84970742223591,
"verified": false
},
{
"type": "manhattan_accuracy",
"value": 87.87014398261343,
"verified": false
},
{
"type": "manhattan_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "manhattan_f1",
"value": 76.31392706820128,
"verified": false
},
{
"type": "manhattan_precision",
"value": 73.91586694566708,
"verified": false
},
{
"type": "manhattan_recall",
"value": 78.87280566676932,
"verified": false
},
{
"type": "max_accuracy",
"value": 87.90119144642372,
"verified": false
},
{
"type": "max_ap",
"value": 84.05164646221583,
"verified": false
},
{
"type": "max_f1",
"value": 76.31392706820128,
"verified": false
}
],
"task": {
"type": "PairClassification"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB VideoRetrieval",
"revision": "58c2597a5943a2ba48f4668c3b90d796283c5639",
"split": "dev",
"type": "C-MTEB/VideoRetrieval"
},
"metrics": [
{
"type": "map_at_1",
"value": 63.6,
"verified": false
},
{
"type": "map_at_10",
"value": 72.673,
"verified": false
},
{
"type": "map_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "map_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "map_at_3",
"value": 70.833,
"verified": false
},
{
"type": "map_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "mrr_at_1",
"value": 63.6,
"verified": false
},
{
"type": "mrr_at_10",
"value": 72.673,
"verified": false
},
{
"type": "mrr_at_100",
"value": 73.05199999999999,
"verified": false
},
{
"type": "mrr_at_1000",
"value": 73.057,
"verified": false
},
{
"type": "mrr_at_3",
"value": 70.833,
"verified": false
},
{
"type": "mrr_at_5",
"value": 72.05799999999999,
"verified": false
},
{
"type": "ndcg_at_1",
"value": 63.6,
"verified": false
},
{
"type": "ndcg_at_10",
"value": 76.776,
"verified": false
},
{
"type": "ndcg_at_100",
"value": 78.52900000000001,
"verified": false
},
{
"type": "ndcg_at_1000",
"value": 78.696,
"verified": false
},
{
"type": "ndcg_at_3",
"value": 73.093,
"verified": false
},
{
"type": "ndcg_at_5",
"value": 75.288,
"verified": false
},
{
"type": "precision_at_1",
"value": 63.6,
"verified": false
},
{
"type": "precision_at_10",
"value": 8.95,
"verified": false
},
{
"type": "precision_at_100",
"value": 0.975,
"verified": false
},
{
"type": "precision_at_1000",
"value": 0.099,
"verified": false
},
{
"type": "precision_at_3",
"value": 26.533,
"verified": false
},
{
"type": "precision_at_5",
"value": 16.98,
"verified": false
},
{
"type": "recall_at_1",
"value": 63.6,
"verified": false
},
{
"type": "recall_at_10",
"value": 89.5,
"verified": false
},
{
"type": "recall_at_100",
"value": 97.5,
"verified": false
},
{
"type": "recall_at_1000",
"value": 98.9,
"verified": false
},
{
"type": "recall_at_3",
"value": 79.60000000000001,
"verified": false
},
{
"type": "recall_at_5",
"value": 84.89999999999999,
"verified": false
}
],
"task": {
"type": "Retrieval"
}
},
{
"dataset": {
"config": "default",
"name": "MTEB Waimai",
"revision": "339287def212450dcaa9df8c22bf93e9980c7023",
"split": "test",
"type": "C-MTEB/waimai-classification"
},
"metrics": [
{
"type": "accuracy",
"value": 89.39999999999999,
"verified": false
},
{
"type": "ap",
"value": 75.52087544076016,
"verified": false
},
{
"type": "f1",
"value": 87.7629629899278,
"verified": false
}
],
"task": {
"type": "Classification"
}
}
]
}
],
"modelId": "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct",
"pipeline_tag": "sentence-similarity",
"private": false,
"safetensors": {
"parameters": {
"F32": 2208985600
},
"total": 2208985600
},
"sha": "9cfa6413f704a7c1cf5064d240748e10c876b286",
"siblings": [
{
"rfilename": ".gitattributes"
},
{
"rfilename": "1_Pooling/config.json"
},
{
"rfilename": "README.md"
},
{
"rfilename": "added_tokens.json"
},
{
"rfilename": "chat_template.json"
},
{
"rfilename": "config.json"
},
{
"rfilename": "config_sentence_transformers.json"
},
{
"rfilename": "custom_st.py"
},
{
"rfilename": "generation_config.json"
},
{
"rfilename": "gme_inference.py"
},
{
"rfilename": "images/gme_logo.png"
},
{
"rfilename": "merges.txt"
},
{
"rfilename": "model-00001-of-00003.safetensors"
},
{
"rfilename": "model-00002-of-00003.safetensors"
},
{
"rfilename": "model-00003-of-00003.safetensors"
},
{
"rfilename": "model.safetensors.index.json"
},
{
"rfilename": "modeling_gme_qwen2vl.py"
},
{
"rfilename": "modules.json"
},
{
"rfilename": "preprocessor_config.json"
},
{
"rfilename": "results.json"
},
{
"rfilename": "special_tokens_map.json"
},
{
"rfilename": "tokenizer.json"
},
{
"rfilename": "tokenizer_config.json"
},
{
"rfilename": "vocab.json"
}
],
"spaces": [
"mteb/leaderboard",
"tardellirs/model-pulse",
"maxpar1/leaderboard",
"kiedistv/leaderboard",
"qiaosong88/leaderboard",
"ArtemBolgov/leaderboard",
"malyvena/leaderboard",
"HuggingLLM/leaderboard",
"wangsh0111/leaderboard",
"kaotd/leaderboard"
],
"tags": [
"sentence-transformers",
"safetensors",
"qwen2_vl",
"image-text-to-text",
"mteb",
"transformers",
"Qwen2-VL",
"sentence-similarity",
"vidore",
"custom_code",
"en",
"zh",
"arxiv:2412.16855",
"base_model:Qwen/Qwen2-VL-2B-Instruct",
"base_model:finetune:Qwen/Qwen2-VL-2B-Instruct",
"license:apache-2.0",
"model-index",
"endpoints_compatible",
"region:us"
],
"transformersInfo": {
"auto_model": "AutoModelForMultimodalLM",
"pipeline_tag": "image-text-to-text",
"processor": "AutoProcessor"
},
"usedStorage": 8836021280,
"widgetData": [
{
"sentences": [
"That is a happy dog",
"That is a very happy person",
"Today is a sunny day"
],
"source_sentence": "That is a happy person"
}
]
} |
Text
This source's terms allow its title, its values and a link here, not its text. It is at https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct.