|
|
--- |
|
|
library_name: sentence-transformers |
|
|
pipeline_tag: sentence-similarity |
|
|
tags: |
|
|
- sentence-transformers |
|
|
- feature-extraction |
|
|
- sentence-similarity |
|
|
- transformers |
|
|
- french |
|
|
- english |
|
|
- sentence-embedding |
|
|
- mteb |
|
|
model-index: |
|
|
- name: 7eff199d41ff669fad99d83cad9249c393c3f14b |
|
|
results: |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: lyon-nlp/alloprof |
|
|
name: MTEB AlloProfClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: 392ba3f5bcc8c51f578786c1fc3dae648662cb9b |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 59.69196295449414 |
|
|
- type: v_measures |
|
|
value: [0.6355772777559684, 0.4980707615440343, 0.5851538838323186, 0.6567709175938427, 0.5712405288636999] |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: lyon-nlp/alloprof |
|
|
name: MTEB AlloProfClusteringS2S |
|
|
config: default |
|
|
split: test |
|
|
revision: 392ba3f5bcc8c51f578786c1fc3dae648662cb9b |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 45.607106996926426 |
|
|
- type: v_measures |
|
|
value: [0.45846869913649535, 0.42657120373128293, 0.45507356125930876, 0.4258913306353704, 0.4779122207000794] |
|
|
- task: |
|
|
type: Reranking |
|
|
dataset: |
|
|
type: lyon-nlp/mteb-fr-reranking-alloprof-s2p |
|
|
name: MTEB AlloprofReranking |
|
|
config: default |
|
|
split: test |
|
|
revision: 65393d0d7a08a10b4e348135e824f385d420b0fd |
|
|
metrics: |
|
|
- type: map |
|
|
value: 73.51836428087765 |
|
|
- type: mrr |
|
|
value: 74.8550285111166 |
|
|
- type: nAUC_map_diff1 |
|
|
value: 56.006169898728466 |
|
|
- type: nAUC_map_max |
|
|
value: 27.886037223407506 |
|
|
- type: nAUC_mrr_diff1 |
|
|
value: 56.68072778248672 |
|
|
- type: nAUC_mrr_max |
|
|
value: 29.362681962243276 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: lyon-nlp/alloprof |
|
|
name: MTEB AlloprofRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: fcf295ea64c750f41fadbaa37b9b861558e1bfbd |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 32.080999999999996 |
|
|
- type: map_at_10 |
|
|
value: 43.582 |
|
|
- type: map_at_100 |
|
|
value: 44.381 |
|
|
- type: map_at_1000 |
|
|
value: 44.426 |
|
|
- type: map_at_20 |
|
|
value: 44.061 |
|
|
- type: map_at_3 |
|
|
value: 40.602 |
|
|
- type: map_at_5 |
|
|
value: 42.381 |
|
|
- type: mrr_at_1 |
|
|
value: 32.08117443868739 |
|
|
- type: mrr_at_10 |
|
|
value: 43.5823429832498 |
|
|
- type: mrr_at_100 |
|
|
value: 44.38068560877513 |
|
|
- type: mrr_at_1000 |
|
|
value: 44.426194305504026 |
|
|
- type: mrr_at_20 |
|
|
value: 44.06128094655753 |
|
|
- type: mrr_at_3 |
|
|
value: 40.60161197466903 |
|
|
- type: mrr_at_5 |
|
|
value: 42.380541162924715 |
|
|
- type: nauc_map_at_1000_diff1 |
|
|
value: 37.22997629352391 |
|
|
- type: nauc_map_at_1000_max |
|
|
value: 38.65090969900466 |
|
|
- type: nauc_map_at_100_diff1 |
|
|
value: 37.22644507166512 |
|
|
- type: nauc_map_at_100_max |
|
|
value: 38.67447923917633 |
|
|
- type: nauc_map_at_10_diff1 |
|
|
value: 37.02440573022942 |
|
|
- type: nauc_map_at_10_max |
|
|
value: 38.52972171430789 |
|
|
- type: nauc_map_at_1_diff1 |
|
|
value: 41.18101653444774 |
|
|
- type: nauc_map_at_1_max |
|
|
value: 34.87383192583458 |
|
|
- type: nauc_map_at_20_diff1 |
|
|
value: 37.14172285932024 |
|
|
- type: nauc_map_at_20_max |
|
|
value: 38.66753159239803 |
|
|
- type: nauc_map_at_3_diff1 |
|
|
value: 37.53556306862998 |
|
|
- type: nauc_map_at_3_max |
|
|
value: 37.86008195327724 |
|
|
- type: nauc_map_at_5_diff1 |
|
|
value: 37.14904081229067 |
|
|
- type: nauc_map_at_5_max |
|
|
value: 38.267819714061105 |
|
|
- type: nauc_mrr_at_1000_diff1 |
|
|
value: 37.22997629352391 |
|
|
- type: nauc_mrr_at_1000_max |
|
|
value: 38.65090969900466 |
|
|
- type: nauc_mrr_at_100_diff1 |
|
|
value: 37.22644507166512 |
|
|
- type: nauc_mrr_at_100_max |
|
|
value: 38.67447923917633 |
|
|
- type: nauc_mrr_at_10_diff1 |
|
|
value: 37.02440573022942 |
|
|
- type: nauc_mrr_at_10_max |
|
|
value: 38.52972171430789 |
|
|
- type: nauc_mrr_at_1_diff1 |
|
|
value: 41.18101653444774 |
|
|
- type: nauc_mrr_at_1_max |
|
|
value: 34.87383192583458 |
|
|
- type: nauc_mrr_at_20_diff1 |
|
|
value: 37.14172285932024 |
|
|
- type: nauc_mrr_at_20_max |
|
|
value: 38.66753159239803 |
|
|
- type: nauc_mrr_at_3_diff1 |
|
|
value: 37.53556306862998 |
|
|
- type: nauc_mrr_at_3_max |
|
|
value: 37.86008195327724 |
|
|
- type: nauc_mrr_at_5_diff1 |
|
|
value: 37.14904081229067 |
|
|
- type: nauc_mrr_at_5_max |
|
|
value: 38.267819714061105 |
|
|
- type: nauc_ndcg_at_1000_diff1 |
|
|
value: 36.313082263552204 |
|
|
- type: nauc_ndcg_at_1000_max |
|
|
value: 40.244406213773765 |
|
|
- type: nauc_ndcg_at_100_diff1 |
|
|
value: 36.17060946689135 |
|
|
- type: nauc_ndcg_at_100_max |
|
|
value: 41.069278488584416 |
|
|
- type: nauc_ndcg_at_10_diff1 |
|
|
value: 35.2775471480974 |
|
|
- type: nauc_ndcg_at_10_max |
|
|
value: 40.33902753007036 |
|
|
- type: nauc_ndcg_at_1_diff1 |
|
|
value: 41.18101653444774 |
|
|
- type: nauc_ndcg_at_1_max |
|
|
value: 34.87383192583458 |
|
|
- type: nauc_ndcg_at_20_diff1 |
|
|
value: 35.71067272175871 |
|
|
- type: nauc_ndcg_at_20_max |
|
|
value: 40.94374381572908 |
|
|
- type: nauc_ndcg_at_3_diff1 |
|
|
value: 36.45082651868188 |
|
|
- type: nauc_ndcg_at_3_max |
|
|
value: 38.87195110158222 |
|
|
- type: nauc_ndcg_at_5_diff1 |
|
|
value: 35.683568481780505 |
|
|
- type: nauc_ndcg_at_5_max |
|
|
value: 39.606933866599 |
|
|
- type: nauc_precision_at_1000_diff1 |
|
|
value: 15.489726515767439 |
|
|
- type: nauc_precision_at_1000_max |
|
|
value: 75.94259161180715 |
|
|
- type: nauc_precision_at_100_diff1 |
|
|
value: 30.033605095284656 |
|
|
- type: nauc_precision_at_100_max |
|
|
value: 62.40786465750442 |
|
|
- type: nauc_precision_at_10_diff1 |
|
|
value: 28.617170969915 |
|
|
- type: nauc_precision_at_10_max |
|
|
value: 47.35884745487521 |
|
|
- type: nauc_precision_at_1_diff1 |
|
|
value: 41.18101653444774 |
|
|
- type: nauc_precision_at_1_max |
|
|
value: 34.87383192583458 |
|
|
- type: nauc_precision_at_20_diff1 |
|
|
value: 29.730952749557144 |
|
|
- type: nauc_precision_at_20_max |
|
|
value: 52.09696741873719 |
|
|
- type: nauc_precision_at_3_diff1 |
|
|
value: 33.30844921569695 |
|
|
- type: nauc_precision_at_3_max |
|
|
value: 41.84496633792437 |
|
|
- type: nauc_precision_at_5_diff1 |
|
|
value: 31.000246292430838 |
|
|
- type: nauc_precision_at_5_max |
|
|
value: 43.88721507465343 |
|
|
- type: nauc_recall_at_1000_diff1 |
|
|
value: 15.48972651576705 |
|
|
- type: nauc_recall_at_1000_max |
|
|
value: 75.94259161180725 |
|
|
- type: nauc_recall_at_100_diff1 |
|
|
value: 30.033605095284816 |
|
|
- type: nauc_recall_at_100_max |
|
|
value: 62.40786465750426 |
|
|
- type: nauc_recall_at_10_diff1 |
|
|
value: 28.617170969914984 |
|
|
- type: nauc_recall_at_10_max |
|
|
value: 47.35884745487525 |
|
|
- type: nauc_recall_at_1_diff1 |
|
|
value: 41.18101653444774 |
|
|
- type: nauc_recall_at_1_max |
|
|
value: 34.87383192583458 |
|
|
- type: nauc_recall_at_20_diff1 |
|
|
value: 29.730952749557087 |
|
|
- type: nauc_recall_at_20_max |
|
|
value: 52.09696741873715 |
|
|
- type: nauc_recall_at_3_diff1 |
|
|
value: 33.30844921569694 |
|
|
- type: nauc_recall_at_3_max |
|
|
value: 41.84496633792433 |
|
|
- type: nauc_recall_at_5_diff1 |
|
|
value: 31.000246292430838 |
|
|
- type: nauc_recall_at_5_max |
|
|
value: 43.88721507465339 |
|
|
- type: ndcg_at_1 |
|
|
value: 32.080999999999996 |
|
|
- type: ndcg_at_10 |
|
|
value: 49.502 |
|
|
- type: ndcg_at_100 |
|
|
value: 53.52 |
|
|
- type: ndcg_at_1000 |
|
|
value: 54.842 |
|
|
- type: ndcg_at_20 |
|
|
value: 51.219 |
|
|
- type: ndcg_at_3 |
|
|
value: 43.381 |
|
|
- type: ndcg_at_5 |
|
|
value: 46.603 |
|
|
- type: precision_at_1 |
|
|
value: 32.080999999999996 |
|
|
- type: precision_at_10 |
|
|
value: 6.822 |
|
|
- type: precision_at_100 |
|
|
value: 0.873 |
|
|
- type: precision_at_1000 |
|
|
value: 0.098 |
|
|
- type: precision_at_20 |
|
|
value: 3.7479999999999998 |
|
|
- type: precision_at_3 |
|
|
value: 17.142 |
|
|
- type: precision_at_5 |
|
|
value: 11.857 |
|
|
- type: recall_at_1 |
|
|
value: 32.080999999999996 |
|
|
- type: recall_at_10 |
|
|
value: 68.221 |
|
|
- type: recall_at_100 |
|
|
value: 87.349 |
|
|
- type: recall_at_1000 |
|
|
value: 98.014 |
|
|
- type: recall_at_20 |
|
|
value: 74.957 |
|
|
- type: recall_at_3 |
|
|
value: 51.425 |
|
|
- type: recall_at_5 |
|
|
value: 59.282999999999994 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_reviews_multi |
|
|
name: MTEB AmazonReviewsClassification (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 39.892 |
|
|
- type: f1 |
|
|
value: 38.38126304364462 |
|
|
- type: f1_weighted |
|
|
value: 38.38126304364462 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: maastrichtlawtech/bsard |
|
|
name: MTEB BSARDRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: 5effa1b9b5fa3b0f9e12523e6e43e5f86a6e6d59 |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 10.811 |
|
|
- type: map_at_10 |
|
|
value: 16.414 |
|
|
- type: map_at_100 |
|
|
value: 17.647 |
|
|
- type: map_at_1000 |
|
|
value: 17.742 |
|
|
- type: map_at_20 |
|
|
value: 17.22 |
|
|
- type: map_at_3 |
|
|
value: 14.188999999999998 |
|
|
- type: map_at_5 |
|
|
value: 15.113 |
|
|
- type: mrr_at_1 |
|
|
value: 10.81081081081081 |
|
|
- type: mrr_at_10 |
|
|
value: 16.41427141427142 |
|
|
- type: mrr_at_100 |
|
|
value: 17.647339314041712 |
|
|
- type: mrr_at_1000 |
|
|
value: 17.74213263983212 |
|
|
- type: mrr_at_20 |
|
|
value: 17.219989884463573 |
|
|
- type: mrr_at_3 |
|
|
value: 14.18918918918919 |
|
|
- type: mrr_at_5 |
|
|
value: 15.112612612612612 |
|
|
- type: nauc_map_at_1000_diff1 |
|
|
value: 13.07108195916555 |
|
|
- type: nauc_map_at_1000_max |
|
|
value: 14.000521014179807 |
|
|
- type: nauc_map_at_100_diff1 |
|
|
value: 13.087117094079332 |
|
|
- type: nauc_map_at_100_max |
|
|
value: 13.99712558752583 |
|
|
- type: nauc_map_at_10_diff1 |
|
|
value: 13.452029501381165 |
|
|
- type: nauc_map_at_10_max |
|
|
value: 13.3341655571542 |
|
|
- type: nauc_map_at_1_diff1 |
|
|
value: 14.990419981155167 |
|
|
- type: nauc_map_at_1_max |
|
|
value: 8.812519082504037 |
|
|
- type: nauc_map_at_20_diff1 |
|
|
value: 12.80321357992737 |
|
|
- type: nauc_map_at_20_max |
|
|
value: 14.020962859032371 |
|
|
- type: nauc_map_at_3_diff1 |
|
|
value: 14.84230805712973 |
|
|
- type: nauc_map_at_3_max |
|
|
value: 11.644032755353722 |
|
|
- type: nauc_map_at_5_diff1 |
|
|
value: 15.100168959732835 |
|
|
- type: nauc_map_at_5_max |
|
|
value: 13.634801099074355 |
|
|
- type: nauc_mrr_at_1000_diff1 |
|
|
value: 13.07108195916555 |
|
|
- type: nauc_mrr_at_1000_max |
|
|
value: 14.000521014179807 |
|
|
- type: nauc_mrr_at_100_diff1 |
|
|
value: 13.087117094079332 |
|
|
- type: nauc_mrr_at_100_max |
|
|
value: 13.99712558752583 |
|
|
- type: nauc_mrr_at_10_diff1 |
|
|
value: 13.452029501381165 |
|
|
- type: nauc_mrr_at_10_max |
|
|
value: 13.3341655571542 |
|
|
- type: nauc_mrr_at_1_diff1 |
|
|
value: 14.990419981155167 |
|
|
- type: nauc_mrr_at_1_max |
|
|
value: 8.812519082504037 |
|
|
- type: nauc_mrr_at_20_diff1 |
|
|
value: 12.80321357992737 |
|
|
- type: nauc_mrr_at_20_max |
|
|
value: 14.020962859032371 |
|
|
- type: nauc_mrr_at_3_diff1 |
|
|
value: 14.84230805712973 |
|
|
- type: nauc_mrr_at_3_max |
|
|
value: 11.644032755353722 |
|
|
- type: nauc_mrr_at_5_diff1 |
|
|
value: 15.100168959732835 |
|
|
- type: nauc_mrr_at_5_max |
|
|
value: 13.634801099074355 |
|
|
- type: nauc_ndcg_at_1000_diff1 |
|
|
value: 11.335350893370972 |
|
|
- type: nauc_ndcg_at_1000_max |
|
|
value: 16.09665875369169 |
|
|
- type: nauc_ndcg_at_100_diff1 |
|
|
value: 11.499643600969176 |
|
|
- type: nauc_ndcg_at_100_max |
|
|
value: 15.967105414704186 |
|
|
- type: nauc_ndcg_at_10_diff1 |
|
|
value: 12.093263549786606 |
|
|
- type: nauc_ndcg_at_10_max |
|
|
value: 14.605821897766461 |
|
|
- type: nauc_ndcg_at_1_diff1 |
|
|
value: 14.990419981155167 |
|
|
- type: nauc_ndcg_at_1_max |
|
|
value: 8.812519082504037 |
|
|
- type: nauc_ndcg_at_20_diff1 |
|
|
value: 10.197380043193812 |
|
|
- type: nauc_ndcg_at_20_max |
|
|
value: 16.332533239525365 |
|
|
- type: nauc_ndcg_at_3_diff1 |
|
|
value: 14.835825175950765 |
|
|
- type: nauc_ndcg_at_3_max |
|
|
value: 11.898757954417214 |
|
|
- type: nauc_ndcg_at_5_diff1 |
|
|
value: 15.278603386081823 |
|
|
- type: nauc_ndcg_at_5_max |
|
|
value: 15.007133861218167 |
|
|
- type: nauc_precision_at_1000_diff1 |
|
|
value: 2.7469897420865195 |
|
|
- type: nauc_precision_at_1000_max |
|
|
value: 26.874535278616346 |
|
|
- type: nauc_precision_at_100_diff1 |
|
|
value: 7.600735526139776 |
|
|
- type: nauc_precision_at_100_max |
|
|
value: 20.7203382946415 |
|
|
- type: nauc_precision_at_10_diff1 |
|
|
value: 8.938642089366768 |
|
|
- type: nauc_precision_at_10_max |
|
|
value: 17.320961743140874 |
|
|
- type: nauc_precision_at_1_diff1 |
|
|
value: 14.990419981155167 |
|
|
- type: nauc_precision_at_1_max |
|
|
value: 8.812519082504037 |
|
|
- type: nauc_precision_at_20_diff1 |
|
|
value: 3.733877816322278 |
|
|
- type: nauc_precision_at_20_max |
|
|
value: 21.581173305923002 |
|
|
- type: nauc_precision_at_3_diff1 |
|
|
value: 14.828850401790316 |
|
|
- type: nauc_precision_at_3_max |
|
|
value: 12.369943286612463 |
|
|
- type: nauc_precision_at_5_diff1 |
|
|
value: 15.728617939150672 |
|
|
- type: nauc_precision_at_5_max |
|
|
value: 18.103783411900697 |
|
|
- type: nauc_recall_at_1000_diff1 |
|
|
value: 2.746989742086615 |
|
|
- type: nauc_recall_at_1000_max |
|
|
value: 26.874535278616367 |
|
|
- type: nauc_recall_at_100_diff1 |
|
|
value: 7.600735526139775 |
|
|
- type: nauc_recall_at_100_max |
|
|
value: 20.720338294641536 |
|
|
- type: nauc_recall_at_10_diff1 |
|
|
value: 8.93864208936673 |
|
|
- type: nauc_recall_at_10_max |
|
|
value: 17.32096174314083 |
|
|
- type: nauc_recall_at_1_diff1 |
|
|
value: 14.990419981155167 |
|
|
- type: nauc_recall_at_1_max |
|
|
value: 8.812519082504037 |
|
|
- type: nauc_recall_at_20_diff1 |
|
|
value: 3.733877816322231 |
|
|
- type: nauc_recall_at_20_max |
|
|
value: 21.58117330592295 |
|
|
- type: nauc_recall_at_3_diff1 |
|
|
value: 14.828850401790339 |
|
|
- type: nauc_recall_at_3_max |
|
|
value: 12.369943286612509 |
|
|
- type: nauc_recall_at_5_diff1 |
|
|
value: 15.72861793915063 |
|
|
- type: nauc_recall_at_5_max |
|
|
value: 18.103783411900658 |
|
|
- type: ndcg_at_1 |
|
|
value: 10.811 |
|
|
- type: ndcg_at_10 |
|
|
value: 20.244 |
|
|
- type: ndcg_at_100 |
|
|
value: 26.526 |
|
|
- type: ndcg_at_1000 |
|
|
value: 29.217 |
|
|
- type: ndcg_at_20 |
|
|
value: 23.122 |
|
|
- type: ndcg_at_3 |
|
|
value: 15.396 |
|
|
- type: ndcg_at_5 |
|
|
value: 17.063 |
|
|
- type: precision_at_1 |
|
|
value: 10.811 |
|
|
- type: precision_at_10 |
|
|
value: 3.288 |
|
|
- type: precision_at_100 |
|
|
value: 0.631 |
|
|
- type: precision_at_1000 |
|
|
value: 0.08499999999999999 |
|
|
- type: precision_at_20 |
|
|
value: 2.207 |
|
|
- type: precision_at_3 |
|
|
value: 6.306000000000001 |
|
|
- type: precision_at_5 |
|
|
value: 4.595 |
|
|
- type: recall_at_1 |
|
|
value: 10.811 |
|
|
- type: recall_at_10 |
|
|
value: 32.883 |
|
|
- type: recall_at_100 |
|
|
value: 63.063 |
|
|
- type: recall_at_1000 |
|
|
value: 84.685 |
|
|
- type: recall_at_20 |
|
|
value: 44.144 |
|
|
- type: recall_at_3 |
|
|
value: 18.919 |
|
|
- type: recall_at_5 |
|
|
value: 22.973 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: lyon-nlp/clustering-hal-s2s |
|
|
name: MTEB HALClusteringS2S |
|
|
config: default |
|
|
split: test |
|
|
revision: e06ebbbb123f8144bef1a5d18796f3dec9ae2915 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 25.209561281028435 |
|
|
- type: v_measures |
|
|
value: [0.28558356565178666, 0.2707322246129254, 0.2683693125038299, 0.2703937853835602, 0.22057190525667872] |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: reciTAL/mlsum |
|
|
name: MTEB MLSUMClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: b5d54f8f3b61ae17845046286940f03c6bc79bc7 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 42.82528809996964 |
|
|
- type: v_measures |
|
|
value: [0.43465029372260205, 0.42821098223656917, 0.43537879149583325, 0.4289578694928627, 0.3794307754465835] |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: reciTAL/mlsum |
|
|
name: MTEB MLSUMClusteringS2S |
|
|
config: default |
|
|
split: test |
|
|
revision: b5d54f8f3b61ae17845046286940f03c6bc79bc7 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 43.44172295073941 |
|
|
- type: v_measures |
|
|
value: [0.4294163918345751, 0.46229994906725164, 0.44188446196569603, 0.43839320352264155, 0.3866853445120933] |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_domain |
|
|
name: MTEB MTOPDomainClassification (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 88.33072345756342 |
|
|
- type: f1 |
|
|
value: 88.11780476022122 |
|
|
- type: f1_weighted |
|
|
value: 88.28188145087299 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_intent |
|
|
name: MTEB MTOPIntentClassification (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 57.854682117131226 |
|
|
- type: f1 |
|
|
value: 41.121569078191996 |
|
|
- type: f1_weighted |
|
|
value: 60.04845437480532 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/masakhanews |
|
|
name: MTEB MasakhaNEWSClassification (fra) |
|
|
config: fra |
|
|
split: test |
|
|
revision: 18193f187b92da67168c655c9973a165ed9593dd |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.87203791469194 |
|
|
- type: f1 |
|
|
value: 72.94847557303437 |
|
|
- type: f1_weighted |
|
|
value: 76.9128173959562 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: masakhane/masakhanews |
|
|
name: MTEB MasakhaNEWSClusteringP2P (fra) |
|
|
config: fra |
|
|
split: test |
|
|
revision: 8ccc72e69e65f40c70e117d8b3c08306bb788b60 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 61.32006896333715 |
|
|
- type: v_measures |
|
|
value: [1.0, 0.6446188396257355, 0.28995363026757603, 0.40898735994696084, 0.7224436183265853] |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: masakhane/masakhanews |
|
|
name: MTEB MasakhaNEWSClusteringS2S (fra) |
|
|
config: fra |
|
|
split: test |
|
|
revision: 8ccc72e69e65f40c70e117d8b3c08306bb788b60 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 60.509887123660256 |
|
|
- type: v_measures |
|
|
value: [1.0, 0.022472587992562534, 0.4686320087689936, 0.811946141094871, 0.7224436183265853] |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: 4672e20407010da34463acc759c162ca9734bca6 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 64.14256893073302 |
|
|
- type: f1 |
|
|
value: 61.33068109342782 |
|
|
- type: f1_weighted |
|
|
value: 62.74292948992287 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 70.68930733019502 |
|
|
- type: f1 |
|
|
value: 70.26641874846638 |
|
|
- type: f1_weighted |
|
|
value: 70.35250466465047 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: jinaai/mintakaqa |
|
|
name: MTEB MintakaRetrieval (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 19.165 |
|
|
- type: map_at_10 |
|
|
value: 28.663 |
|
|
- type: map_at_100 |
|
|
value: 29.737000000000002 |
|
|
- type: map_at_1000 |
|
|
value: 29.826000000000004 |
|
|
- type: map_at_20 |
|
|
value: 29.266 |
|
|
- type: map_at_3 |
|
|
value: 26.024 |
|
|
- type: map_at_5 |
|
|
value: 27.486 |
|
|
- type: mrr_at_1 |
|
|
value: 19.164619164619165 |
|
|
- type: mrr_at_10 |
|
|
value: 28.66298116298116 |
|
|
- type: mrr_at_100 |
|
|
value: 29.737423308510476 |
|
|
- type: mrr_at_1000 |
|
|
value: 29.825744096186796 |
|
|
- type: mrr_at_20 |
|
|
value: 29.26593905045215 |
|
|
- type: mrr_at_3 |
|
|
value: 26.023751023751025 |
|
|
- type: mrr_at_5 |
|
|
value: 27.48566748566751 |
|
|
- type: nauc_map_at_1000_diff1 |
|
|
value: 23.682512151202967 |
|
|
- type: nauc_map_at_1000_max |
|
|
value: 25.78708364723919 |
|
|
- type: nauc_map_at_100_diff1 |
|
|
value: 23.647360144907324 |
|
|
- type: nauc_map_at_100_max |
|
|
value: 25.812420160707074 |
|
|
- type: nauc_map_at_10_diff1 |
|
|
value: 23.658224717435765 |
|
|
- type: nauc_map_at_10_max |
|
|
value: 25.845198626323217 |
|
|
- type: nauc_map_at_1_diff1 |
|
|
value: 30.56830621718086 |
|
|
- type: nauc_map_at_1_max |
|
|
value: 19.931526248650147 |
|
|
- type: nauc_map_at_20_diff1 |
|
|
value: 23.69662048930091 |
|
|
- type: nauc_map_at_20_max |
|
|
value: 25.936653022318403 |
|
|
- type: nauc_map_at_3_diff1 |
|
|
value: 24.663221072349817 |
|
|
- type: nauc_map_at_3_max |
|
|
value: 24.634011858800275 |
|
|
- type: nauc_map_at_5_diff1 |
|
|
value: 24.3650772668551 |
|
|
- type: nauc_map_at_5_max |
|
|
value: 25.75222318469224 |
|
|
- type: nauc_mrr_at_1000_diff1 |
|
|
value: 23.682512151202967 |
|
|
- type: nauc_mrr_at_1000_max |
|
|
value: 25.78708364723919 |
|
|
- type: nauc_mrr_at_100_diff1 |
|
|
value: 23.647360144907324 |
|
|
- type: nauc_mrr_at_100_max |
|
|
value: 25.812420160707074 |
|
|
- type: nauc_mrr_at_10_diff1 |
|
|
value: 23.658224717435765 |
|
|
- type: nauc_mrr_at_10_max |
|
|
value: 25.845198626323217 |
|
|
- type: nauc_mrr_at_1_diff1 |
|
|
value: 30.56830621718086 |
|
|
- type: nauc_mrr_at_1_max |
|
|
value: 19.931526248650147 |
|
|
- type: nauc_mrr_at_20_diff1 |
|
|
value: 23.69662048930091 |
|
|
- type: nauc_mrr_at_20_max |
|
|
value: 25.936653022318403 |
|
|
- type: nauc_mrr_at_3_diff1 |
|
|
value: 24.663221072349817 |
|
|
- type: nauc_mrr_at_3_max |
|
|
value: 24.634011858800275 |
|
|
- type: nauc_mrr_at_5_diff1 |
|
|
value: 24.3650772668551 |
|
|
- type: nauc_mrr_at_5_max |
|
|
value: 25.75222318469224 |
|
|
- type: nauc_ndcg_at_1000_diff1 |
|
|
value: 21.68690756038845 |
|
|
- type: nauc_ndcg_at_1000_max |
|
|
value: 27.168575101114893 |
|
|
- type: nauc_ndcg_at_100_diff1 |
|
|
value: 20.484812648526646 |
|
|
- type: nauc_ndcg_at_100_max |
|
|
value: 27.79987215383081 |
|
|
- type: nauc_ndcg_at_10_diff1 |
|
|
value: 20.791330920997765 |
|
|
- type: nauc_ndcg_at_10_max |
|
|
value: 28.272774035036935 |
|
|
- type: nauc_ndcg_at_1_diff1 |
|
|
value: 30.56830621718086 |
|
|
- type: nauc_ndcg_at_1_max |
|
|
value: 19.931526248650147 |
|
|
- type: nauc_ndcg_at_20_diff1 |
|
|
value: 20.88342749790573 |
|
|
- type: nauc_ndcg_at_20_max |
|
|
value: 28.627184419546825 |
|
|
- type: nauc_ndcg_at_3_diff1 |
|
|
value: 22.987235018840494 |
|
|
- type: nauc_ndcg_at_3_max |
|
|
value: 26.054144215976482 |
|
|
- type: nauc_ndcg_at_5_diff1 |
|
|
value: 22.497863289090464 |
|
|
- type: nauc_ndcg_at_5_max |
|
|
value: 27.98879570850259 |
|
|
- type: nauc_precision_at_1000_diff1 |
|
|
value: -0.6707404502167996 |
|
|
- type: nauc_precision_at_1000_max |
|
|
value: 31.987217077673346 |
|
|
- type: nauc_precision_at_100_diff1 |
|
|
value: 5.079765403021014 |
|
|
- type: nauc_precision_at_100_max |
|
|
value: 34.857053312543194 |
|
|
- type: nauc_precision_at_10_diff1 |
|
|
value: 12.628771618059472 |
|
|
- type: nauc_precision_at_10_max |
|
|
value: 35.009564954169896 |
|
|
- type: nauc_precision_at_1_diff1 |
|
|
value: 30.56830621718086 |
|
|
- type: nauc_precision_at_1_max |
|
|
value: 19.931526248650147 |
|
|
- type: nauc_precision_at_20_diff1 |
|
|
value: 12.28251326261041 |
|
|
- type: nauc_precision_at_20_max |
|
|
value: 36.942629359432075 |
|
|
- type: nauc_precision_at_3_diff1 |
|
|
value: 18.663775283519335 |
|
|
- type: nauc_precision_at_3_max |
|
|
value: 29.741315837492472 |
|
|
- type: nauc_precision_at_5_diff1 |
|
|
value: 17.70442691217025 |
|
|
- type: nauc_precision_at_5_max |
|
|
value: 33.93438470540527 |
|
|
- type: nauc_recall_at_1000_diff1 |
|
|
value: -0.6707404502171719 |
|
|
- type: nauc_recall_at_1000_max |
|
|
value: 31.987217077672607 |
|
|
- type: nauc_recall_at_100_diff1 |
|
|
value: 5.079765403021056 |
|
|
- type: nauc_recall_at_100_max |
|
|
value: 34.85705331254323 |
|
|
- type: nauc_recall_at_10_diff1 |
|
|
value: 12.628771618059483 |
|
|
- type: nauc_recall_at_10_max |
|
|
value: 35.00956495416992 |
|
|
- type: nauc_recall_at_1_diff1 |
|
|
value: 30.56830621718086 |
|
|
- type: nauc_recall_at_1_max |
|
|
value: 19.931526248650147 |
|
|
- type: nauc_recall_at_20_diff1 |
|
|
value: 12.282513262610411 |
|
|
- type: nauc_recall_at_20_max |
|
|
value: 36.94262935943207 |
|
|
- type: nauc_recall_at_3_diff1 |
|
|
value: 18.663775283519346 |
|
|
- type: nauc_recall_at_3_max |
|
|
value: 29.741315837492465 |
|
|
- type: nauc_recall_at_5_diff1 |
|
|
value: 17.704426912170252 |
|
|
- type: nauc_recall_at_5_max |
|
|
value: 33.934384705405286 |
|
|
- type: ndcg_at_1 |
|
|
value: 19.165 |
|
|
- type: ndcg_at_10 |
|
|
value: 33.674 |
|
|
- type: ndcg_at_100 |
|
|
value: 39.297 |
|
|
- type: ndcg_at_1000 |
|
|
value: 41.896 |
|
|
- type: ndcg_at_20 |
|
|
value: 35.842 |
|
|
- type: ndcg_at_3 |
|
|
value: 28.238999999999997 |
|
|
- type: ndcg_at_5 |
|
|
value: 30.863000000000003 |
|
|
- type: precision_at_1 |
|
|
value: 19.165 |
|
|
- type: precision_at_10 |
|
|
value: 4.9590000000000005 |
|
|
- type: precision_at_100 |
|
|
value: 0.768 |
|
|
- type: precision_at_1000 |
|
|
value: 0.098 |
|
|
- type: precision_at_20 |
|
|
value: 2.905 |
|
|
- type: precision_at_3 |
|
|
value: 11.548 |
|
|
- type: precision_at_5 |
|
|
value: 8.198 |
|
|
- type: recall_at_1 |
|
|
value: 19.165 |
|
|
- type: recall_at_10 |
|
|
value: 49.59 |
|
|
- type: recall_at_100 |
|
|
value: 76.822 |
|
|
- type: recall_at_1000 |
|
|
value: 97.83 |
|
|
- type: recall_at_20 |
|
|
value: 58.108000000000004 |
|
|
- type: recall_at_3 |
|
|
value: 34.644000000000005 |
|
|
- type: recall_at_5 |
|
|
value: 40.991 |
|
|
- task: |
|
|
type: PairClassification |
|
|
dataset: |
|
|
type: GEM/opusparcus |
|
|
name: MTEB OpusparcusPC (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: 9e9b1f8ef51616073f47f306f7f47dd91663f86a |
|
|
metrics: |
|
|
- type: cos_sim_accuracy |
|
|
value: 83.51498637602179 |
|
|
- type: cos_sim_ap |
|
|
value: 94.18614574224773 |
|
|
- type: cos_sim_f1 |
|
|
value: 88.3564925730714 |
|
|
- type: cos_sim_precision |
|
|
value: 85.37037037037037 |
|
|
- type: cos_sim_recall |
|
|
value: 91.55908639523337 |
|
|
- type: dot_accuracy |
|
|
value: 83.51498637602179 |
|
|
- type: dot_ap |
|
|
value: 94.18614574224773 |
|
|
- type: dot_f1 |
|
|
value: 88.3564925730714 |
|
|
- type: dot_precision |
|
|
value: 85.37037037037037 |
|
|
- type: dot_recall |
|
|
value: 91.55908639523337 |
|
|
- type: euclidean_accuracy |
|
|
value: 83.51498637602179 |
|
|
- type: euclidean_ap |
|
|
value: 94.18614574224773 |
|
|
- type: euclidean_f1 |
|
|
value: 88.3564925730714 |
|
|
- type: euclidean_precision |
|
|
value: 85.37037037037037 |
|
|
- type: euclidean_recall |
|
|
value: 91.55908639523337 |
|
|
- type: manhattan_accuracy |
|
|
value: 83.51498637602179 |
|
|
- type: manhattan_ap |
|
|
value: 94.16717671332795 |
|
|
- type: manhattan_f1 |
|
|
value: 88.35418671799807 |
|
|
- type: manhattan_precision |
|
|
value: 85.71428571428571 |
|
|
- type: manhattan_recall |
|
|
value: 91.16186693147964 |
|
|
- type: max_accuracy |
|
|
value: 83.51498637602179 |
|
|
- type: max_ap |
|
|
value: 94.18614574224773 |
|
|
- type: max_f1 |
|
|
value: 88.3564925730714 |
|
|
- task: |
|
|
type: PairClassification |
|
|
dataset: |
|
|
type: google-research-datasets/paws-x |
|
|
name: MTEB PawsX (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: 8a04d940a42cd40658986fdd8e3da561533a3646 |
|
|
metrics: |
|
|
- type: cos_sim_accuracy |
|
|
value: 60.699999999999996 |
|
|
- type: cos_sim_ap |
|
|
value: 60.20276173325004 |
|
|
- type: cos_sim_f1 |
|
|
value: 62.716429395921516 |
|
|
- type: cos_sim_precision |
|
|
value: 48.05424528301887 |
|
|
- type: cos_sim_recall |
|
|
value: 90.2547065337763 |
|
|
- type: dot_accuracy |
|
|
value: 60.699999999999996 |
|
|
- type: dot_ap |
|
|
value: 60.27996470746299 |
|
|
- type: dot_f1 |
|
|
value: 62.716429395921516 |
|
|
- type: dot_precision |
|
|
value: 48.05424528301887 |
|
|
- type: dot_recall |
|
|
value: 90.2547065337763 |
|
|
- type: euclidean_accuracy |
|
|
value: 60.699999999999996 |
|
|
- type: euclidean_ap |
|
|
value: 60.20276173325004 |
|
|
- type: euclidean_f1 |
|
|
value: 62.716429395921516 |
|
|
- type: euclidean_precision |
|
|
value: 48.05424528301887 |
|
|
- type: euclidean_recall |
|
|
value: 90.2547065337763 |
|
|
- type: manhattan_accuracy |
|
|
value: 60.699999999999996 |
|
|
- type: manhattan_ap |
|
|
value: 60.18010040913353 |
|
|
- type: manhattan_f1 |
|
|
value: 62.71056661562021 |
|
|
- type: manhattan_precision |
|
|
value: 47.92276184903452 |
|
|
- type: manhattan_recall |
|
|
value: 90.69767441860465 |
|
|
- type: max_accuracy |
|
|
value: 60.699999999999996 |
|
|
- type: max_ap |
|
|
value: 60.27996470746299 |
|
|
- type: max_f1 |
|
|
value: 62.716429395921516 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: Lajavaness/SICK-fr |
|
|
name: MTEB SICKFr |
|
|
config: default |
|
|
split: test |
|
|
revision: e077ab4cf4774a1e36d86d593b150422fafd8e8a |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 84.24496945719946 |
|
|
- type: cos_sim_spearman |
|
|
value: 78.10001513346513 |
|
|
- type: euclidean_pearson |
|
|
value: 81.43570951228163 |
|
|
- type: euclidean_spearman |
|
|
value: 78.0987784421045 |
|
|
- type: manhattan_pearson |
|
|
value: 81.31986646517238 |
|
|
- type: manhattan_spearman |
|
|
value: 78.09610194828534 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts22-crosslingual-sts |
|
|
name: MTEB STS22 (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 83.07721141521425 |
|
|
- type: cos_sim_spearman |
|
|
value: 83.19199466052186 |
|
|
- type: euclidean_pearson |
|
|
value: 82.10672022294766 |
|
|
- type: euclidean_spearman |
|
|
value: 83.19199466052186 |
|
|
- type: manhattan_pearson |
|
|
value: 81.92531847793633 |
|
|
- type: manhattan_spearman |
|
|
value: 83.20694689089673 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/stsb_multi_mt |
|
|
name: MTEB STSBenchmarkMultilingualSTS (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: 29afa2569dcedaaa2fe6a3dcfebab33d28b82e8c |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 83.957481748094 |
|
|
- type: cos_sim_spearman |
|
|
value: 84.40492503459248 |
|
|
- type: euclidean_pearson |
|
|
value: 83.8150014101056 |
|
|
- type: euclidean_spearman |
|
|
value: 84.40686653864509 |
|
|
- type: manhattan_pearson |
|
|
value: 83.6816837321264 |
|
|
- type: manhattan_spearman |
|
|
value: 84.2678486368702 |
|
|
- task: |
|
|
type: Summarization |
|
|
dataset: |
|
|
type: lyon-nlp/summarization-summeval-fr-p2p |
|
|
name: MTEB SummEvalFr |
|
|
config: default |
|
|
split: test |
|
|
revision: b385812de6a9577b6f4d0f88c6a6e35395a94054 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 32.06592630917136 |
|
|
- type: cos_sim_spearman |
|
|
value: 30.94878864229808 |
|
|
- type: dot_pearson |
|
|
value: 32.06591974515864 |
|
|
- type: dot_spearman |
|
|
value: 30.925383080565222 |
|
|
- task: |
|
|
type: Reranking |
|
|
dataset: |
|
|
type: lyon-nlp/mteb-fr-reranking-syntec-s2p |
|
|
name: MTEB SyntecReranking |
|
|
config: default |
|
|
split: test |
|
|
revision: daf0863838cd9e3ba50544cdce3ac2b338a1b0ad |
|
|
metrics: |
|
|
- type: map |
|
|
value: 88.11666666666667 |
|
|
- type: mrr |
|
|
value: 88.11666666666667 |
|
|
- type: nAUC_map_diff1 |
|
|
value: 66.27779227667267 |
|
|
- type: nAUC_map_max |
|
|
value: 6.651414764738896 |
|
|
- type: nAUC_mrr_diff1 |
|
|
value: 66.27779227667267 |
|
|
- type: nAUC_mrr_max |
|
|
value: 6.651414764738896 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: lyon-nlp/mteb-fr-retrieval-syntec-s2p |
|
|
name: MTEB SyntecRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: 19661ccdca4dfc2d15122d776b61685f48c68ca9 |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 69.0 |
|
|
- type: map_at_10 |
|
|
value: 80.65 |
|
|
- type: map_at_100 |
|
|
value: 80.838 |
|
|
- type: map_at_1000 |
|
|
value: 80.838 |
|
|
- type: map_at_20 |
|
|
value: 80.838 |
|
|
- type: map_at_3 |
|
|
value: 79.833 |
|
|
- type: map_at_5 |
|
|
value: 80.483 |
|
|
- type: mrr_at_1 |
|
|
value: 69.0 |
|
|
- type: mrr_at_10 |
|
|
value: 80.64999999999999 |
|
|
- type: mrr_at_100 |
|
|
value: 80.83799019607844 |
|
|
- type: mrr_at_1000 |
|
|
value: 80.83799019607844 |
|
|
- type: mrr_at_20 |
|
|
value: 80.83799019607844 |
|
|
- type: mrr_at_3 |
|
|
value: 79.83333333333334 |
|
|
- type: mrr_at_5 |
|
|
value: 80.48333333333333 |
|
|
- type: nauc_map_at_1000_diff1 |
|
|
value: 61.46904865740055 |
|
|
- type: nauc_map_at_1000_max |
|
|
value: 24.307826758747282 |
|
|
- type: nauc_map_at_100_diff1 |
|
|
value: 61.46904865740055 |
|
|
- type: nauc_map_at_100_max |
|
|
value: 24.307826758747282 |
|
|
- type: nauc_map_at_10_diff1 |
|
|
value: 61.094194035098035 |
|
|
- type: nauc_map_at_10_max |
|
|
value: 24.44687875369869 |
|
|
- type: nauc_map_at_1_diff1 |
|
|
value: 65.17628798701865 |
|
|
- type: nauc_map_at_1_max |
|
|
value: 25.79501560929155 |
|
|
- type: nauc_map_at_20_diff1 |
|
|
value: 61.46904865740055 |
|
|
- type: nauc_map_at_20_max |
|
|
value: 24.307826758747282 |
|
|
- type: nauc_map_at_3_diff1 |
|
|
value: 61.562719756100805 |
|
|
- type: nauc_map_at_3_max |
|
|
value: 25.87804164282553 |
|
|
- type: nauc_map_at_5_diff1 |
|
|
value: 61.471976470716264 |
|
|
- type: nauc_map_at_5_max |
|
|
value: 25.180513270581322 |
|
|
- type: nauc_mrr_at_1000_diff1 |
|
|
value: 61.46904865740055 |
|
|
- type: nauc_mrr_at_1000_max |
|
|
value: 24.307826758747282 |
|
|
- type: nauc_mrr_at_100_diff1 |
|
|
value: 61.46904865740055 |
|
|
- type: nauc_mrr_at_100_max |
|
|
value: 24.307826758747282 |
|
|
- type: nauc_mrr_at_10_diff1 |
|
|
value: 61.094194035098035 |
|
|
- type: nauc_mrr_at_10_max |
|
|
value: 24.44687875369869 |
|
|
- type: nauc_mrr_at_1_diff1 |
|
|
value: 65.17628798701865 |
|
|
- type: nauc_mrr_at_1_max |
|
|
value: 25.79501560929155 |
|
|
- type: nauc_mrr_at_20_diff1 |
|
|
value: 61.46904865740055 |
|
|
- type: nauc_mrr_at_20_max |
|
|
value: 24.307826758747282 |
|
|
- type: nauc_mrr_at_3_diff1 |
|
|
value: 61.562719756100805 |
|
|
- type: nauc_mrr_at_3_max |
|
|
value: 25.87804164282553 |
|
|
- type: nauc_mrr_at_5_diff1 |
|
|
value: 61.471976470716264 |
|
|
- type: nauc_mrr_at_5_max |
|
|
value: 25.180513270581322 |
|
|
- type: nauc_ndcg_at_1000_diff1 |
|
|
value: 60.95477865546023 |
|
|
- type: nauc_ndcg_at_1000_max |
|
|
value: 24.427553593893535 |
|
|
- type: nauc_ndcg_at_100_diff1 |
|
|
value: 60.95477865546023 |
|
|
- type: nauc_ndcg_at_100_max |
|
|
value: 24.427553593893535 |
|
|
- type: nauc_ndcg_at_10_diff1 |
|
|
value: 59.101673931307396 |
|
|
- type: nauc_ndcg_at_10_max |
|
|
value: 25.01155211084955 |
|
|
- type: nauc_ndcg_at_1_diff1 |
|
|
value: 65.17628798701865 |
|
|
- type: nauc_ndcg_at_1_max |
|
|
value: 25.79501560929155 |
|
|
- type: nauc_ndcg_at_20_diff1 |
|
|
value: 60.95477865546023 |
|
|
- type: nauc_ndcg_at_20_max |
|
|
value: 24.427553593893535 |
|
|
- type: nauc_ndcg_at_3_diff1 |
|
|
value: 60.333057480044616 |
|
|
- type: nauc_ndcg_at_3_max |
|
|
value: 28.363238330232637 |
|
|
- type: nauc_ndcg_at_5_diff1 |
|
|
value: 60.15511994533307 |
|
|
- type: nauc_ndcg_at_5_max |
|
|
value: 26.94308058940176 |
|
|
- type: nauc_precision_at_1000_diff1 |
|
|
value: nan |
|
|
- type: nauc_precision_at_1000_max |
|
|
value: nan |
|
|
- type: nauc_precision_at_100_diff1 |
|
|
value: nan |
|
|
- type: nauc_precision_at_100_max |
|
|
value: nan |
|
|
- type: nauc_precision_at_10_diff1 |
|
|
value: 26.657329598506518 |
|
|
- type: nauc_precision_at_10_max |
|
|
value: 34.26704014939361 |
|
|
- type: nauc_precision_at_1_diff1 |
|
|
value: 65.17628798701865 |
|
|
- type: nauc_precision_at_1_max |
|
|
value: 25.79501560929155 |
|
|
- type: nauc_precision_at_20_diff1 |
|
|
value: 100.0 |
|
|
- type: nauc_precision_at_20_max |
|
|
value: 100.0 |
|
|
- type: nauc_precision_at_3_diff1 |
|
|
value: 51.834066960117276 |
|
|
- type: nauc_precision_at_3_max |
|
|
value: 48.25930372148875 |
|
|
- type: nauc_precision_at_5_diff1 |
|
|
value: 44.992997198879706 |
|
|
- type: nauc_precision_at_5_max |
|
|
value: 50.70028011204499 |
|
|
- type: nauc_recall_at_1000_diff1 |
|
|
value: nan |
|
|
- type: nauc_recall_at_1000_max |
|
|
value: nan |
|
|
- type: nauc_recall_at_100_diff1 |
|
|
value: nan |
|
|
- type: nauc_recall_at_100_max |
|
|
value: nan |
|
|
- type: nauc_recall_at_10_diff1 |
|
|
value: 26.657329598505903 |
|
|
- type: nauc_recall_at_10_max |
|
|
value: 34.26704014939303 |
|
|
- type: nauc_recall_at_1_diff1 |
|
|
value: 65.17628798701865 |
|
|
- type: nauc_recall_at_1_max |
|
|
value: 25.79501560929155 |
|
|
- type: nauc_recall_at_20_diff1 |
|
|
value: nan |
|
|
- type: nauc_recall_at_20_max |
|
|
value: nan |
|
|
- type: nauc_recall_at_3_diff1 |
|
|
value: 51.834066960117376 |
|
|
- type: nauc_recall_at_3_max |
|
|
value: 48.25930372148865 |
|
|
- type: nauc_recall_at_5_diff1 |
|
|
value: 44.99299719887955 |
|
|
- type: nauc_recall_at_5_max |
|
|
value: 50.70028011204488 |
|
|
- type: ndcg_at_1 |
|
|
value: 69.0 |
|
|
- type: ndcg_at_10 |
|
|
value: 84.786 |
|
|
- type: ndcg_at_100 |
|
|
value: 85.521 |
|
|
- type: ndcg_at_1000 |
|
|
value: 85.521 |
|
|
- type: ndcg_at_20 |
|
|
value: 85.521 |
|
|
- type: ndcg_at_3 |
|
|
value: 83.226 |
|
|
- type: ndcg_at_5 |
|
|
value: 84.43 |
|
|
- type: precision_at_1 |
|
|
value: 69.0 |
|
|
- type: precision_at_10 |
|
|
value: 9.700000000000001 |
|
|
- type: precision_at_100 |
|
|
value: 1.0 |
|
|
- type: precision_at_1000 |
|
|
value: 0.1 |
|
|
- type: precision_at_20 |
|
|
value: 5.0 |
|
|
- type: precision_at_3 |
|
|
value: 31.0 |
|
|
- type: precision_at_5 |
|
|
value: 19.2 |
|
|
- type: recall_at_1 |
|
|
value: 69.0 |
|
|
- type: recall_at_10 |
|
|
value: 97.0 |
|
|
- type: recall_at_100 |
|
|
value: 100.0 |
|
|
- type: recall_at_1000 |
|
|
value: 100.0 |
|
|
- type: recall_at_20 |
|
|
value: 100.0 |
|
|
- type: recall_at_3 |
|
|
value: 93.0 |
|
|
- type: recall_at_5 |
|
|
value: 96.0 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: jinaai/xpqa |
|
|
name: MTEB XPQARetrieval (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 40.797 |
|
|
- type: map_at_10 |
|
|
value: 62.71099999999999 |
|
|
- type: map_at_100 |
|
|
value: 64.261 |
|
|
- type: map_at_1000 |
|
|
value: 64.306 |
|
|
- type: map_at_20 |
|
|
value: 63.693 |
|
|
- type: map_at_3 |
|
|
value: 56.686 |
|
|
- type: map_at_5 |
|
|
value: 60.653999999999996 |
|
|
- type: mrr_at_1 |
|
|
value: 64.08544726301736 |
|
|
- type: mrr_at_10 |
|
|
value: 71.24790726259349 |
|
|
- type: mrr_at_100 |
|
|
value: 71.7835679704396 |
|
|
- type: mrr_at_1000 |
|
|
value: 71.79095567140973 |
|
|
- type: mrr_at_20 |
|
|
value: 71.5854708410262 |
|
|
- type: mrr_at_3 |
|
|
value: 69.55941255006672 |
|
|
- type: mrr_at_5 |
|
|
value: 70.60747663551396 |
|
|
- type: nauc_map_at_1000_diff1 |
|
|
value: 47.803181417639365 |
|
|
- type: nauc_map_at_1000_max |
|
|
value: 51.22073368230412 |
|
|
- type: nauc_map_at_100_diff1 |
|
|
value: 47.771573391555755 |
|
|
- type: nauc_map_at_100_max |
|
|
value: 51.20370234778812 |
|
|
- type: nauc_map_at_10_diff1 |
|
|
value: 47.340833389771625 |
|
|
- type: nauc_map_at_10_max |
|
|
value: 50.41256517180715 |
|
|
- type: nauc_map_at_1_diff1 |
|
|
value: 55.14983744702445 |
|
|
- type: nauc_map_at_1_max |
|
|
value: 31.104750896985728 |
|
|
- type: nauc_map_at_20_diff1 |
|
|
value: 47.64026863999484 |
|
|
- type: nauc_map_at_20_max |
|
|
value: 50.87670909266768 |
|
|
- type: nauc_map_at_3_diff1 |
|
|
value: 47.681906747352635 |
|
|
- type: nauc_map_at_3_max |
|
|
value: 43.47246277661219 |
|
|
- type: nauc_map_at_5_diff1 |
|
|
value: 46.874943002794815 |
|
|
- type: nauc_map_at_5_max |
|
|
value: 48.469495140739724 |
|
|
- type: nauc_mrr_at_1000_diff1 |
|
|
value: 57.34098736669957 |
|
|
- type: nauc_mrr_at_1000_max |
|
|
value: 60.179095583193444 |
|
|
- type: nauc_mrr_at_100_diff1 |
|
|
value: 57.339862158018796 |
|
|
- type: nauc_mrr_at_100_max |
|
|
value: 60.18082273539442 |
|
|
- type: nauc_mrr_at_10_diff1 |
|
|
value: 57.210874058908814 |
|
|
- type: nauc_mrr_at_10_max |
|
|
value: 60.043680803697086 |
|
|
- type: nauc_mrr_at_1_diff1 |
|
|
value: 59.69074056197331 |
|
|
- type: nauc_mrr_at_1_max |
|
|
value: 60.90082316300324 |
|
|
- type: nauc_mrr_at_20_diff1 |
|
|
value: 57.35434243512763 |
|
|
- type: nauc_mrr_at_20_max |
|
|
value: 60.18873377253912 |
|
|
- type: nauc_mrr_at_3_diff1 |
|
|
value: 57.26933631425754 |
|
|
- type: nauc_mrr_at_3_max |
|
|
value: 60.05458089795687 |
|
|
- type: nauc_mrr_at_5_diff1 |
|
|
value: 57.045411517214276 |
|
|
- type: nauc_mrr_at_5_max |
|
|
value: 59.981421712413685 |
|
|
- type: nauc_ndcg_at_1000_diff1 |
|
|
value: 50.232929738614814 |
|
|
- type: nauc_ndcg_at_1000_max |
|
|
value: 55.01594185277396 |
|
|
- type: nauc_ndcg_at_100_diff1 |
|
|
value: 49.876825728406786 |
|
|
- type: nauc_ndcg_at_100_max |
|
|
value: 54.87898182661215 |
|
|
- type: nauc_ndcg_at_10_diff1 |
|
|
value: 48.40787615482867 |
|
|
- type: nauc_ndcg_at_10_max |
|
|
value: 52.84877289626636 |
|
|
- type: nauc_ndcg_at_1_diff1 |
|
|
value: 59.69074056197331 |
|
|
- type: nauc_ndcg_at_1_max |
|
|
value: 60.90082316300324 |
|
|
- type: nauc_ndcg_at_20_diff1 |
|
|
value: 49.08453974591539 |
|
|
- type: nauc_ndcg_at_20_max |
|
|
value: 53.80319392912378 |
|
|
- type: nauc_ndcg_at_3_diff1 |
|
|
value: 48.21830414023458 |
|
|
- type: nauc_ndcg_at_3_max |
|
|
value: 51.321799626032714 |
|
|
- type: nauc_ndcg_at_5_diff1 |
|
|
value: 47.614495954542605 |
|
|
- type: nauc_ndcg_at_5_max |
|
|
value: 50.803800463597405 |
|
|
- type: nauc_precision_at_1000_diff1 |
|
|
value: -15.87250509394414 |
|
|
- type: nauc_precision_at_1000_max |
|
|
value: 16.09830137145176 |
|
|
- type: nauc_precision_at_100_diff1 |
|
|
value: -13.720930651556534 |
|
|
- type: nauc_precision_at_100_max |
|
|
value: 19.94363871765946 |
|
|
- type: nauc_precision_at_10_diff1 |
|
|
value: -3.9626074014054136 |
|
|
- type: nauc_precision_at_10_max |
|
|
value: 30.48732389685921 |
|
|
- type: nauc_precision_at_1_diff1 |
|
|
value: 59.69074056197331 |
|
|
- type: nauc_precision_at_1_max |
|
|
value: 60.90082316300324 |
|
|
- type: nauc_precision_at_20_diff1 |
|
|
value: -8.144148640034853 |
|
|
- type: nauc_precision_at_20_max |
|
|
value: 26.183545158653338 |
|
|
- type: nauc_precision_at_3_diff1 |
|
|
value: 7.1166818076254605 |
|
|
- type: nauc_precision_at_3_max |
|
|
value: 37.64665636029093 |
|
|
- type: nauc_precision_at_5_diff1 |
|
|
value: 0.3455996928663316 |
|
|
- type: nauc_precision_at_5_max |
|
|
value: 34.95245204298077 |
|
|
- type: nauc_recall_at_1000_diff1 |
|
|
value: 47.93171740380228 |
|
|
- type: nauc_recall_at_1000_max |
|
|
value: 89.21354057542635 |
|
|
- type: nauc_recall_at_100_diff1 |
|
|
value: 34.93973412699365 |
|
|
- type: nauc_recall_at_100_max |
|
|
value: 47.89216950421148 |
|
|
- type: nauc_recall_at_10_diff1 |
|
|
value: 38.58556368247737 |
|
|
- type: nauc_recall_at_10_max |
|
|
value: 45.13227163006313 |
|
|
- type: nauc_recall_at_1_diff1 |
|
|
value: 55.14983744702445 |
|
|
- type: nauc_recall_at_1_max |
|
|
value: 31.104750896985728 |
|
|
- type: nauc_recall_at_20_diff1 |
|
|
value: 38.53568097509877 |
|
|
- type: nauc_recall_at_20_max |
|
|
value: 46.37328875121808 |
|
|
- type: nauc_recall_at_3_diff1 |
|
|
value: 41.49659886305561 |
|
|
- type: nauc_recall_at_3_max |
|
|
value: 38.59476562231703 |
|
|
- type: nauc_recall_at_5_diff1 |
|
|
value: 38.489499442628016 |
|
|
- type: nauc_recall_at_5_max |
|
|
value: 43.06848825600403 |
|
|
- type: ndcg_at_1 |
|
|
value: 64.08500000000001 |
|
|
- type: ndcg_at_10 |
|
|
value: 68.818 |
|
|
- type: ndcg_at_100 |
|
|
value: 73.66 |
|
|
- type: ndcg_at_1000 |
|
|
value: 74.309 |
|
|
- type: ndcg_at_20 |
|
|
value: 71.147 |
|
|
- type: ndcg_at_3 |
|
|
value: 64.183 |
|
|
- type: ndcg_at_5 |
|
|
value: 65.668 |
|
|
- type: precision_at_1 |
|
|
value: 64.08500000000001 |
|
|
- type: precision_at_10 |
|
|
value: 15.728 |
|
|
- type: precision_at_100 |
|
|
value: 1.9720000000000002 |
|
|
- type: precision_at_1000 |
|
|
value: 0.207 |
|
|
- type: precision_at_20 |
|
|
value: 8.705 |
|
|
- type: precision_at_3 |
|
|
value: 39.03 |
|
|
- type: precision_at_5 |
|
|
value: 27.717000000000002 |
|
|
- type: recall_at_1 |
|
|
value: 40.797 |
|
|
- type: recall_at_10 |
|
|
value: 77.432 |
|
|
- type: recall_at_100 |
|
|
value: 95.68100000000001 |
|
|
- type: recall_at_1000 |
|
|
value: 99.666 |
|
|
- type: recall_at_20 |
|
|
value: 84.773 |
|
|
- type: recall_at_3 |
|
|
value: 62.083 |
|
|
- type: recall_at_5 |
|
|
value: 69.786 |
|
|
license: apache-2.0 |
|
|
language: |
|
|
- fr |
|
|
- en |
|
|
--- |
|
|
## Model Description: |
|
|
[**french-document-embedding**](https://huggingface.co/dangvantuan/french-document-embedding) is an embedding model for documents in the French-English language, with a context length of up to 8096 tokens. This model is a specialized text-embedding model trained specifically for the French-English language. It is built upon [gte-multilingual](Alibaba-NLP/gte-multilingual-base) and trained using the [SimilarityLoss], [Multi-Negative Ranking Loss](https://arxiv.org/abs/1705.00652), [Matryoshka2dLoss](https://arxiv.org/html/2402.14776v1) and [GISTEmbedLoss](https://arxiv.org/abs/2402.16829) using [guide model](https://huggingface.co/Lajavaness/bilingual-embedding-large). This model embeds and converts long texts or documents into vectors with 786 dimensions, making it useful for vector databases serving semantic search or RAG (Retrieval-Augmented Generation). |
|
|
|
|
|
## Full Model Architecture |
|
|
``` |
|
|
SentenceTransformer( |
|
|
(0): Transformer({'max_seq_length': 8192, 'do_lower_case': False}) with Transformer model: BilingualModel |
|
|
(1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': True, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True}) |
|
|
(2): Normalize() |
|
|
) |
|
|
``` |
|
|
|
|
|
|
|
|
## Usage: |
|
|
|
|
|
Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed: |
|
|
|
|
|
``` |
|
|
pip install -U sentence-transformers |
|
|
``` |
|
|
|
|
|
Then you can use the model like this: |
|
|
|
|
|
```python |
|
|
from sentence_transformers import SentenceTransformer |
|
|
sentences = ["Paris est une capitale de la France", "Paris is a capital of France"] |
|
|
|
|
|
|
|
|
|
|
|
model = SentenceTransformer('dangvantuan/french-document-embedding', trust_remote_code=True) |
|
|
embeddings = model.encode(sentences) |
|
|
print(embeddings) |
|
|
|
|
|
``` |
|
|
|
|
|
|
|
|
## Evaluation |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
## Citation |
|
|
|
|
|
|
|
|
@article{reimers2019sentence, |
|
|
title={Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks}, |
|
|
author={Nils Reimers, Iryna Gurevych}, |
|
|
journal={https://arxiv.org/abs/1908.10084}, |
|
|
year={2019} |
|
|
} |
|
|
|
|
|
@article{zhang2024mgte, |
|
|
title={mGTE: Generalized Long-Context Text Representation and Reranking Models for Multilingual Text Retrieval}, |
|
|
author={Zhang, Xin and Zhang, Yanzhao and Long, Dingkun and Xie, Wen and Dai, Ziqi and Tang, Jialong and Lin, Huan and Yang, Baosong and Xie, Pengjun and Huang, Fei and others}, |
|
|
journal={arXiv preprint arXiv:2407.19669}, |
|
|
year={2024} |
|
|
} |
|
|
|
|
|
@article{li2023towards, |
|
|
title={Towards general text embeddings with multi-stage contrastive learning}, |
|
|
author={Li, Zehan and Zhang, Xin and Zhang, Yanzhao and Long, Dingkun and Xie, Pengjun and Zhang, Meishan}, |
|
|
journal={arXiv preprint arXiv:2308.03281}, |
|
|
year={2023} |
|
|
} |
|
|
|
|
|
@article{li20242d, |
|
|
title={2d matryoshka sentence embeddings}, |
|
|
author={Li, Xianming and Li, Zongxi and Li, Jing and Xie, Haoran and Li, Qing}, |
|
|
journal={arXiv preprint arXiv:2402.14776}, |
|
|
year={2024} |
|
|
} |
|
|
|
|
|
@misc{henderson2017efficient, |
|
|
title={Efficient Natural Language Response Suggestion for Smart Reply}, |
|
|
author={Matthew Henderson and Rami Al-Rfou and Brian Strope and Yun-hsuan Sung and Laszlo Lukacs and Ruiqi Guo and Sanjiv Kumar and Balint Miklos and Ray Kurzweil}, |
|
|
year={2017}, |
|
|
eprint={1705.00652}, |
|
|
archivePrefix={arXiv}, |
|
|
primaryClass={cs.CL} |
|
|
} |
|
|
|
|
|
@misc{solatorio2024gistembed, |
|
|
title={GISTEmbed: Guided In-sample Selection of Training Negatives for Text Embedding Fine-tuning}, |
|
|
author={Aivin V. Solatorio}, |
|
|
year={2024}, |
|
|
eprint={2402.16829}, |
|
|
archivePrefix={arXiv}, |
|
|
primaryClass={cs.LG} |
|
|
} |
