Upload folder using huggingface_hub
Browse files- .gitattributes +1 -0
- README.md +562 -0
- config.json +36 -0
- model.safetensors +3 -0
- modeling_xprovence_hf.py +499 -0
- sentencepiece.bpe.model +3 -0
- special_tokens_map.json +51 -0
- tokenizer.json +3 -0
- tokenizer_config.json +56 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,562 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html class="">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8" />
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no" />
|
| 6 |
+
<meta name="description" content="We’re on a journey to advance and democratize artificial intelligence through open source and open science." />
|
| 7 |
+
<meta property="fb:app_id" content="1321688464574422" />
|
| 8 |
+
<meta name="twitter:card" content="summary_large_image" />
|
| 9 |
+
<meta name="twitter:site" content="@huggingface" />
|
| 10 |
+
<meta name="twitter:image" content="https://cdn-thumbnails.huggingface.co/social-thumbnails/models/youssef101/xprovence.png" />
|
| 11 |
+
<meta property="og:title" content="README.md · youssef101/xprovence at main" />
|
| 12 |
+
<meta property="og:type" content="website" />
|
| 13 |
+
<meta property="og:url" content="https://huggingface.co/youssef101/xprovence/blob/main/README.md" />
|
| 14 |
+
<meta property="og:image" content="https://cdn-thumbnails.huggingface.co/social-thumbnails/models/youssef101/xprovence.png" />
|
| 15 |
+
|
| 16 |
+
<link rel="stylesheet" href="/front/build/kube-ad7df0d/style.css" />
|
| 17 |
+
|
| 18 |
+
<link rel="preconnect" href="https://fonts.gstatic.com" />
|
| 19 |
+
<link
|
| 20 |
+
href="https://fonts.googleapis.com/css2?family=Source+Sans+Pro:ital,wght@0,200;0,300;0,400;0,600;0,700;1,200;1,300;1,400;1,600;1,700&display=swap"
|
| 21 |
+
rel="stylesheet"
|
| 22 |
+
/>
|
| 23 |
+
<link
|
| 24 |
+
href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;600;700&display=swap"
|
| 25 |
+
rel="stylesheet"
|
| 26 |
+
/>
|
| 27 |
+
|
| 28 |
+
<link
|
| 29 |
+
rel="preload"
|
| 30 |
+
href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.12.0/katex.min.css"
|
| 31 |
+
as="style"
|
| 32 |
+
onload="this.onload=null;this.rel='stylesheet'"
|
| 33 |
+
/>
|
| 34 |
+
<noscript>
|
| 35 |
+
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.12.0/katex.min.css" />
|
| 36 |
+
</noscript>
|
| 37 |
+
|
| 38 |
+
<script>const guestTheme = document.cookie.match(/theme=(\w+)/)?.[1]; document.documentElement.classList.toggle('dark', guestTheme === 'dark' || ( (!guestTheme || guestTheme === 'system') && window.matchMedia('(prefers-color-scheme: dark)').matches));</script>
|
| 39 |
+
<link rel="canonical" href="https://huggingface.co/youssef101/xprovence/blob/main/README.md">
|
| 40 |
+
|
| 41 |
+
<title>README.md · youssef101/xprovence at main</title>
|
| 42 |
+
|
| 43 |
+
<script
|
| 44 |
+
defer
|
| 45 |
+
data-domain="huggingface.co"
|
| 46 |
+
event-loggedIn="false"
|
| 47 |
+
src="/js/script.pageview-props.js"
|
| 48 |
+
></script>
|
| 49 |
+
<script>
|
| 50 |
+
window.plausible =
|
| 51 |
+
window.plausible ||
|
| 52 |
+
function () {
|
| 53 |
+
(window.plausible.q = window.plausible.q || []).push(arguments);
|
| 54 |
+
};
|
| 55 |
+
</script>
|
| 56 |
+
<script>
|
| 57 |
+
window.hubConfig = {"features":{"signupDisabled":false},"sshGitUrl":"[email protected]","moonHttpUrl":"https:\/\/huggingface.co","captchaApiKey":"bd5f2066-93dc-4bdd-a64b-a24646ca3859","captchaDisabledOnSignup":true,"datasetViewerPublicUrl":"https:\/\/datasets-server.huggingface.co","stripePublicKey":"pk_live_x2tdjFXBCvXo2FFmMybezpeM00J6gPCAAc","environment":"production","userAgent":"HuggingFace (production)","spacesIframeDomain":"hf.space","spacesApiUrl":"https:\/\/api.hf.space","docSearchKey":"ece5e02e57300e17d152c08056145326e90c4bff3dd07d7d1ae40cf1c8d39cb6","logoDev":{"apiUrl":"https:\/\/img.logo.dev\/","apiKey":"pk_UHS2HZOeRnaSOdDp7jbd5w"}};
|
| 58 |
+
</script>
|
| 59 |
+
<script type="text/javascript" src="https://de5282c3ca0c.edge.sdk.awswaf.com/de5282c3ca0c/526cf06acb0d/challenge.js" defer></script>
|
| 60 |
+
</head>
|
| 61 |
+
<body class="flex flex-col min-h-dvh bg-white dark:bg-gray-950 text-black ViewerBlobPage">
|
| 62 |
+
<div class="flex min-h-dvh flex-col"><div class="SVELTE_HYDRATER contents" data-target="SystemThemeMonitor" data-props="{"isLoggedIn":false}"></div>
|
| 63 |
+
|
| 64 |
+
<div class="SVELTE_HYDRATER contents" data-target="MainHeader" data-props="{"classNames":"","isWide":false,"isZh":false,"isPro":false}"><header class="border-b border-gray-100 "><div class="w-full px-4 container flex h-16 items-center"><div class="flex flex-1 items-center"><a class="mr-5 flex flex-none items-center lg:mr-6" href="/"><img alt="Hugging Face's logo" class="w-7 md:mr-2" src="/front/assets/huggingface_logo-noborder.svg">
|
| 65 |
+
<span class="hidden whitespace-nowrap text-lg font-bold md:block">Hugging Face</span></a>
|
| 66 |
+
<div class="relative flex-1 lg:max-w-sm mr-2 sm:mr-4 md:mr-3 xl:mr-6"><input autocomplete="off" class="w-full dark:bg-gray-950 pl-8 form-input-alt h-9 pr-3 focus:shadow-xl " name="" placeholder="Search models, datasets, users..." spellcheck="false" type="text" value="">
|
| 67 |
+
<svg class="absolute left-2.5 text-gray-400 top-1/2 transform -translate-y-1/2" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M30 28.59L22.45 21A11 11 0 1 0 21 22.45L28.59 30zM5 14a9 9 0 1 1 9 9a9 9 0 0 1-9-9z" fill="currentColor"></path></svg>
|
| 68 |
+
</div>
|
| 69 |
+
<div class="flex flex-none items-center justify-center p-0.5 place-self-stretch lg:hidden"><button class="relative z-40 flex h-6 w-8 items-center justify-center" type="button"><svg width="1em" height="1em" viewBox="0 0 10 10" class="text-xl" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" preserveAspectRatio="xMidYMid meet" fill="currentColor"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.65039 2.9999C1.65039 2.8066 1.80709 2.6499 2.00039 2.6499H8.00039C8.19369 2.6499 8.35039 2.8066 8.35039 2.9999C8.35039 3.1932 8.19369 3.3499 8.00039 3.3499H2.00039C1.80709 3.3499 1.65039 3.1932 1.65039 2.9999ZM1.65039 4.9999C1.65039 4.8066 1.80709 4.6499 2.00039 4.6499H8.00039C8.19369 4.6499 8.35039 4.8066 8.35039 4.9999C8.35039 5.1932 8.19369 5.3499 8.00039 5.3499H2.00039C1.80709 5.3499 1.65039 5.1932 1.65039 4.9999ZM2.00039 6.6499C1.80709 6.6499 1.65039 6.8066 1.65039 6.9999C1.65039 7.1932 1.80709 7.3499 2.00039 7.3499H8.00039C8.19369 7.3499 8.35039 7.1932 8.35039 6.9999C8.35039 6.8066 8.19369 6.6499 8.00039 6.6499H2.00039Z"></path></svg>
|
| 70 |
+
</button>
|
| 71 |
+
|
| 72 |
+
</div></div>
|
| 73 |
+
<nav aria-label="Main" class="ml-auto hidden lg:block"><ul class="flex items-center gap-x-1 2xl:gap-x-2"><li class="hover:text-indigo-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/models"><svg class="mr-1.5 text-gray-400 group-hover:text-indigo-500" style="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg>
|
| 74 |
+
Models</a>
|
| 75 |
+
</li><li class="hover:text-red-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/datasets"><svg class="mr-1.5 text-gray-400 group-hover:text-red-500" style="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 25 25"><ellipse cx="12.5" cy="5" fill="currentColor" fill-opacity="0.25" rx="7.5" ry="2"></ellipse><path d="M12.5 15C16.6421 15 20 14.1046 20 13V20C20 21.1046 16.6421 22 12.5 22C8.35786 22 5 21.1046 5 20V13C5 14.1046 8.35786 15 12.5 15Z" fill="currentColor" opacity="0.5"></path><path d="M12.5 7C16.6421 7 20 6.10457 20 5V11.5C20 12.6046 16.6421 13.5 12.5 13.5C8.35786 13.5 5 12.6046 5 11.5V5C5 6.10457 8.35786 7 12.5 7Z" fill="currentColor" opacity="0.5"></path><path d="M5.23628 12C5.08204 12.1598 5 12.8273 5 13C5 14.1046 8.35786 15 12.5 15C16.6421 15 20 14.1046 20 13C20 12.8273 19.918 12.1598 19.7637 12C18.9311 12.8626 15.9947 13.5 12.5 13.5C9.0053 13.5 6.06886 12.8626 5.23628 12Z" fill="currentColor"></path></svg>
|
| 76 |
+
Datasets</a>
|
| 77 |
+
</li><li class="hover:text-blue-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/spaces"><svg class="mr-1.5 text-gray-400 group-hover:text-blue-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" viewBox="0 0 25 25"><path opacity=".5" d="M6.016 14.674v4.31h4.31v-4.31h-4.31ZM14.674 14.674v4.31h4.31v-4.31h-4.31ZM6.016 6.016v4.31h4.31v-4.31h-4.31Z" fill="currentColor"></path><path opacity=".75" fill-rule="evenodd" clip-rule="evenodd" d="M3 4.914C3 3.857 3.857 3 4.914 3h6.514c.884 0 1.628.6 1.848 1.414a5.171 5.171 0 0 1 7.31 7.31c.815.22 1.414.964 1.414 1.848v6.514A1.914 1.914 0 0 1 20.086 22H4.914A1.914 1.914 0 0 1 3 20.086V4.914Zm3.016 1.102v4.31h4.31v-4.31h-4.31Zm0 12.968v-4.31h4.31v4.31h-4.31Zm8.658 0v-4.31h4.31v4.31h-4.31Zm0-10.813a2.155 2.155 0 1 1 4.31 0 2.155 2.155 0 0 1-4.31 0Z" fill="currentColor"></path><path opacity=".25" d="M16.829 6.016a2.155 2.155 0 1 0 0 4.31 2.155 2.155 0 0 0 0-4.31Z" fill="currentColor"></path></svg>
|
| 78 |
+
Spaces</a>
|
| 79 |
+
</li><li class="max-xl:hidden relative"><div class="relative ">
|
| 80 |
+
<button class="group flex items-center px-2 py-0.5 dark:text-gray-300 hover:text-yellow-700 dark:hover:text-gray-100 " type="button">
|
| 81 |
+
<svg class="mr-1.5 mr-1.5 text-gray-400 text-yellow-500! group-hover:text-yellow-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M20.6081 3C21.7684 3 22.8053 3.49196 23.5284 4.38415C23.9756 4.93678 24.4428 5.82749 24.4808 7.16133C24.9674 7.01707 25.4353 6.93643 25.8725 6.93643C26.9833 6.93643 27.9865 7.37587 28.696 8.17411C29.6075 9.19872 30.0124 10.4579 29.8361 11.7177C29.7523 12.3177 29.5581 12.8555 29.2678 13.3534C29.8798 13.8646 30.3306 14.5763 30.5485 15.4322C30.719 16.1032 30.8939 17.5006 29.9808 18.9403C30.0389 19.0342 30.0934 19.1319 30.1442 19.2318C30.6932 20.3074 30.7283 21.5229 30.2439 22.6548C29.5093 24.3704 27.6841 25.7219 24.1397 27.1727C21.9347 28.0753 19.9174 28.6523 19.8994 28.6575C16.9842 29.4379 14.3477 29.8345 12.0653 29.8345C7.87017 29.8345 4.8668 28.508 3.13831 25.8921C0.356375 21.6797 0.754104 17.8269 4.35369 14.1131C6.34591 12.058 7.67023 9.02782 7.94613 8.36275C8.50224 6.39343 9.97271 4.20438 12.4172 4.20438H12.4179C12.6236 4.20438 12.8314 4.2214 13.0364 4.25468C14.107 4.42854 15.0428 5.06476 15.7115 6.02205C16.4331 5.09583 17.134 4.359 17.7682 3.94323C18.7242 3.31737 19.6794 3 20.6081 3ZM20.6081 5.95917C20.2427 5.95917 19.7963 6.1197 19.3039 6.44225C17.7754 7.44319 14.8258 12.6772 13.7458 14.7131C13.3839 15.3952 12.7655 15.6837 12.2086 15.6837C11.1036 15.6837 10.2408 14.5497 12.1076 13.1085C14.9146 10.9402 13.9299 7.39584 12.5898 7.1776C12.5311 7.16799 12.4731 7.16355 12.4172 7.16355C11.1989 7.16355 10.6615 9.33114 10.6615 9.33114C10.6615 9.33114 9.0863 13.4148 6.38031 16.206C3.67434 18.998 3.5346 21.2388 5.50675 24.2246C6.85185 26.2606 9.42666 26.8753 12.0653 26.8753C14.8021 26.8753 17.6077 26.2139 19.1799 25.793C19.2574 25.7723 28.8193 22.984 27.6081 20.6107C27.4046 20.212 27.0693 20.0522 26.6471 20.0522C24.9416 20.0522 21.8393 22.6726 20.5057 22.6726C20.2076 22.6726 19.9976 22.5416 19.9116 22.222C19.3433 20.1173 28.552 19.2325 27.7758 16.1839C27.639 15.6445 27.2677 15.4256 26.746 15.4263C24.4923 15.4263 19.4358 19.5181 18.3759 19.5181C18.2949 19.5181 18.2368 19.4937 18.2053 19.4419C17.6743 18.557 17.9653 17.9394 21.7082 15.6009C25.4511 13.2617 28.0783 11.8545 26.5841 10.1752C26.4121 9.98141 26.1684 9.8956 25.8725 9.8956C23.6001 9.89634 18.2311 14.9403 18.2311 14.9403C18.2311 14.9403 16.7821 16.496 15.9057 16.496C15.7043 16.496 15.533 16.4139 15.4169 16.2112C14.7956 15.1296 21.1879 10.1286 21.5484 8.06535C21.7928 6.66715 21.3771 5.95917 20.6081 5.95917Z" fill="#FF9D00"></path><path d="M5.50686 24.2246C3.53472 21.2387 3.67446 18.9979 6.38043 16.206C9.08641 13.4147 10.6615 9.33111 10.6615 9.33111C10.6615 9.33111 11.2499 6.95933 12.59 7.17757C13.93 7.39581 14.9139 10.9401 12.1069 13.1084C9.29997 15.276 12.6659 16.7489 13.7459 14.713C14.8258 12.6772 17.7747 7.44316 19.304 6.44221C20.8326 5.44128 21.9089 6.00204 21.5484 8.06532C21.188 10.1286 14.795 15.1295 15.4171 16.2118C16.0391 17.2934 18.2312 14.9402 18.2312 14.9402C18.2312 14.9402 25.0907 8.49588 26.5842 10.1752C28.0776 11.8545 25.4512 13.2616 21.7082 15.6008C17.9646 17.9393 17.6744 18.557 18.2054 19.4418C18.7372 20.3266 26.9998 13.1351 27.7759 16.1838C28.5513 19.2324 19.3434 20.1173 19.9117 22.2219C20.48 24.3274 26.3979 18.2382 27.6082 20.6107C28.8193 22.9839 19.2574 25.7722 19.18 25.7929C16.0914 26.62 8.24723 28.3726 5.50686 24.2246Z" fill="#FFD21E"></path></svg>
|
| 82 |
+
Community
|
| 83 |
+
</button>
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
</div>
|
| 87 |
+
</li><li class="hover:text-yellow-700"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/docs"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="mr-1.5 text-gray-400 group-hover:text-yellow-500" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 16 16"><path d="m2.28 3.7-.3.16a.67.67 0 0 0-.34.58v8.73l.01.04.02.07.01.04.03.06.02.04.02.03.04.06.05.05.04.04.06.04.06.04.08.04.08.02h.05l.07.02h.11l.04-.01.07-.02.03-.01.07-.03.22-.12a5.33 5.33 0 0 1 5.15.1.67.67 0 0 0 .66 0 5.33 5.33 0 0 1 5.33 0 .67.67 0 0 0 1-.58V4.36a.67.67 0 0 0-.34-.5l-.3-.17v7.78a.63.63 0 0 1-.87.59 4.9 4.9 0 0 0-4.35.35l-.65.39a.29.29 0 0 1-.15.04.29.29 0 0 1-.16-.04l-.65-.4a4.9 4.9 0 0 0-4.34-.34.63.63 0 0 1-.87-.59V3.7Z" fill="currentColor" class="dark:opacity-40"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M8 3.1a5.99 5.99 0 0 0-5.3-.43.66.66 0 0 0-.42.62v8.18c0 .45.46.76.87.59a4.9 4.9 0 0 1 4.34.35l.65.39c.05.03.1.04.16.04.05 0 .1-.01.15-.04l.65-.4a4.9 4.9 0 0 1 4.35-.34.63.63 0 0 0 .86-.59V3.3a.67.67 0 0 0-.41-.62 5.99 5.99 0 0 0-5.3.43l-.3.17L8 3.1Zm.73 1.87a.43.43 0 1 0-.86 0v5.48a.43.43 0 0 0 .86 0V4.97Z" fill="currentColor" class="opacity-40 dark:opacity-100"></path><path d="M8.73 4.97a.43.43 0 1 0-.86 0v5.48a.43.43 0 1 0 .86 0V4.96Z" fill="currentColor" class="dark:opacity-40"></path></svg>
|
| 88 |
+
Docs</a>
|
| 89 |
+
</li><li class="hover:text-black dark:hover:text-white max-2xl:hidden"><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/enterprise"><svg class="mr-1.5 text-gray-400 group-hover:text-black dark:group-hover:text-white" xmlns="http://www.w3.org/2000/svg" fill="none" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 12 12"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.9 1.35a3.16 3.16 0 0 0-2.8 2.07L.37 8.58C0 9.71.7 10.65 1.86 10.65H7.3a3.2 3.2 0 0 0 2.84-2.07l1.67-5.16c.36-1.13-.3-2.07-1.46-2.07H4.91Zm.4 2.07L3.57 8.47h3.57l.36-1.12H5.4l.28-.91h1.75l.4-1.1H6.07l.3-.83h2l.36-1.1H5.27h.04Z" fill="currentColor"></path></svg>
|
| 90 |
+
Enterprise</a>
|
| 91 |
+
</li>
|
| 92 |
+
|
| 93 |
+
<li><a class="group flex items-center px-2 py-0.5 dark:text-gray-300 dark:hover:text-gray-100" href="/pricing">Pricing
|
| 94 |
+
</a></li>
|
| 95 |
+
|
| 96 |
+
<li><div class="relative group">
|
| 97 |
+
<button class="px-2 py-0.5 hover:text-gray-500 dark:hover:text-gray-600 flex items-center " type="button">
|
| 98 |
+
<svg class=" text-gray-500 w-5 group-hover:text-gray-400 dark:text-gray-300 dark:group-hover:text-gray-100" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" viewBox="0 0 32 18" preserveAspectRatio="xMidYMid meet"><path fill-rule="evenodd" clip-rule="evenodd" d="M14.4504 3.30221C14.4504 2.836 14.8284 2.45807 15.2946 2.45807H28.4933C28.9595 2.45807 29.3374 2.836 29.3374 3.30221C29.3374 3.76842 28.9595 4.14635 28.4933 4.14635H15.2946C14.8284 4.14635 14.4504 3.76842 14.4504 3.30221Z" fill="currentColor"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M14.4504 9.00002C14.4504 8.53382 14.8284 8.15588 15.2946 8.15588H28.4933C28.9595 8.15588 29.3374 8.53382 29.3374 9.00002C29.3374 9.46623 28.9595 9.84417 28.4933 9.84417H15.2946C14.8284 9.84417 14.4504 9.46623 14.4504 9.00002Z" fill="currentColor"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M14.4504 14.6978C14.4504 14.2316 14.8284 13.8537 15.2946 13.8537H28.4933C28.9595 13.8537 29.3374 14.2316 29.3374 14.6978C29.3374 15.164 28.9595 15.542 28.4933 15.542H15.2946C14.8284 15.542 14.4504 15.164 14.4504 14.6978Z" fill="currentColor"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M1.94549 6.87377C2.27514 6.54411 2.80962 6.54411 3.13928 6.87377L6.23458 9.96907L9.32988 6.87377C9.65954 6.54411 10.194 6.54411 10.5237 6.87377C10.8533 7.20343 10.8533 7.73791 10.5237 8.06756L6.23458 12.3567L1.94549 8.06756C1.61583 7.73791 1.61583 7.20343 1.94549 6.87377Z" fill="currentColor"></path></svg>
|
| 99 |
+
|
| 100 |
+
</button>
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
</div></li>
|
| 104 |
+
<li><hr class="h-5 w-0.5 border-none bg-gray-100 dark:bg-gray-800"></li>
|
| 105 |
+
<li><a class="block cursor-pointer whitespace-nowrap px-2 py-0.5 hover:text-gray-500 dark:text-gray-300 dark:hover:text-gray-100" href="/login">Log In
|
| 106 |
+
</a></li>
|
| 107 |
+
<li><a class="whitespace-nowrap rounded-full border border-transparent bg-gray-900 px-3 py-1 leading-none text-white hover:border-black hover:bg-white hover:text-black" href="/join">Sign Up
|
| 108 |
+
</a></li></ul></nav></div></header></div>
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
<div class="SVELTE_HYDRATER contents" data-target="SSOBanner" data-props="{}"></div>
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
<main class="flex flex-1 flex-col">
|
| 118 |
+
<div class="SVELTE_HYDRATER contents" data-target="ModelHeader" data-props="{"activeTab":"files","author":{"_id":"631dea73b07bce5f2a7cfdc4","avatarUrl":"/avatars/f504ed0767b1bfdfc550075e477657a8.svg","fullname":"mohamed","name":"youssef101","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false},"canReadRepoSettings":false,"canWriteRepoContent":false,"canDisable":false,"model":{"author":"youssef101","cardData":{"library_name":"transformers","tags":["XProvence","RAG","Multilingual","Provence"],"license":"cc-by-nc-nd-4.0","language":["ar","en","fr","es","bn","fa","fi","hi","id","ja","ko","ru","sw","te","th","zh"],"base_model":["BAAI/bge-reranker-v2-m3"],"pipeline_tag":"text-ranking"},"cardExists":true,"config":{"architectures":["XProvence"],"model_type":"xlm-roberta","auto_map":{"AutoModel":"modeling_xprovence_hf.XProvence"},"tokenizer_config":{"bos_token":"<s>","cls_token":"<s>","eos_token":"</s>","mask_token":"<mask>","pad_token":"<pad>","sep_token":"</s>","unk_token":"<unk>"}},"createdAt":"2025-08-26T14:34:16.000Z","discussionsDisabled":false,"downloads":14,"downloadsAllTime":14,"id":"youssef101/xprovence","isLikedByUser":false,"availableInferenceProviders":[],"inference":"","lastModified":"2025-09-04T11:14:55.000Z","likes":0,"pipeline_tag":"text-ranking","library_name":"transformers","librariesOther":[],"trackDownloads":true,"mask_token":"<mask>","model-index":null,"private":false,"repoType":"model","gated":"manual","pwcLink":{"error":"Unknown error, can't generate link to Papers With Code."},"tags":["transformers","safetensors","xlm-roberta","feature-extraction","XProvence","RAG","Multilingual","Provence","text-ranking","custom_code","ar","en","fr","es","bn","fa","fi","hi","id","ja","ko","ru","sw","te","th","zh","base_model:BAAI/bge-reranker-v2-m3","base_model:finetune:BAAI/bge-reranker-v2-m3","license:cc-by-nc-nd-4.0","region:us"],"tag_objs":[{"id":"text-ranking","label":"Text Ranking","type":"pipeline_tag","subType":"nlp"},{"id":"transformers","label":"Transformers","type":"library"},{"id":"safetensors","label":"Safetensors","type":"library"},{"id":"ar","label":"Arabic","type":"language"},{"id":"en","label":"English","type":"language"},{"id":"fr","label":"French","type":"language"},{"id":"es","label":"Spanish","type":"language"},{"id":"bn","label":"Bengali","type":"language"},{"id":"fa","label":"Persian","type":"language"},{"id":"fi","label":"Finnish","type":"language"},{"id":"hi","label":"Hindi","type":"language"},{"id":"id","label":"Indonesian","type":"language"},{"id":"ja","label":"Japanese","type":"language"},{"id":"ko","label":"Korean","type":"language"},{"id":"ru","label":"Russian","type":"language"},{"id":"sw","label":"Swahili","type":"language"},{"id":"te","label":"Telugu","type":"language"},{"id":"th","label":"Thai","type":"language"},{"id":"zh","label":"Chinese","type":"language"},{"id":"xlm-roberta","label":"xlm-roberta","type":"other"},{"id":"feature-extraction","label":"feature-extraction","type":"other"},{"id":"XProvence","label":"XProvence","type":"other"},{"id":"RAG","label":"RAG","type":"other"},{"id":"Multilingual","label":"Multilingual","type":"other"},{"id":"Provence","label":"Provence","type":"other"},{"id":"custom_code","label":"custom_code","type":"other"},{"id":"base_model:BAAI/bge-reranker-v2-m3","label":"base_model:BAAI/bge-reranker-v2-m3","type":"other"},{"id":"base_model:finetune:BAAI/bge-reranker-v2-m3","label":"base_model:finetune:BAAI/bge-reranker-v2-m3","type":"other"},{"id":"license:cc-by-nc-nd-4.0","label":"cc-by-nc-nd-4.0","type":"license"},{"type":"region","label":"🇺🇸 Region: US","id":"region:us"}],"transformersInfo":{"auto_model":"AutoModel","custom_class":"modeling_xprovence_hf.XProvence","pipeline_tag":"feature-extraction","processor":"AutoTokenizer"},"safetensors":{"parameters":{"F32":567757827},"total":567757827,"sharded":false},"hasBlockedOids":false,"region":"us","isQuantized":false,"xetEnabled":true},"discussionsStats":{"closed":0,"open":0,"total":0},"query":{},"inferenceContextData":{"billableEntities":[],"entityName2Providers":{}}}"><header class="bg-linear-to-t border-b border-gray-100 pt-6 sm:pt-9 from-purple-500/8 dark:from-purple-500/20 to-white to-70% dark:to-gray-950"><div class="container relative "><h1 class="flex flex-wrap items-center max-md:leading-tight mb-3 text-lg max-sm:gap-y-1.5 md:text-xl">
|
| 119 |
+
<div class="group flex flex-none items-center"><div class="relative mr-1 flex items-center">
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
<span class="inline-block "><span class="contents"><a href="/youssef101" class="text-gray-400 hover:text-blue-600"><img alt="" class="size-3.5 rounded-full flex-none" src="/avatars/f504ed0767b1bfdfc550075e477657a8.svg" crossorigin="anonymous"></a></span>
|
| 124 |
+
</span></div>
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
<span class="inline-block "><span class="contents"><a href="/youssef101" class="text-gray-400 hover:text-blue-600">youssef101</a></span>
|
| 128 |
+
</span>
|
| 129 |
+
<div class="mx-0.5 text-gray-300">/</div></div>
|
| 130 |
+
|
| 131 |
+
<div class="max-w-full "><a class="break-words font-mono font-semibold hover:text-blue-600 " href="/youssef101/xprovence">xprovence</a>
|
| 132 |
+
<button class="text-sm mr-4 focus:outline-hidden inline-flex cursor-pointer items-center text-sm mx-0.5 text-gray-600 " title="Copy model name to clipboard" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
|
| 133 |
+
</button></div>
|
| 134 |
+
<div class="inline-flex items-center overflow-hidden whitespace-nowrap rounded-md border bg-white text-sm leading-none text-gray-500 mr-2"><button class="relative flex items-center overflow-hidden from-red-50 to-transparent dark:from-red-900 px-1.5 py-1 hover:bg-linear-to-t focus:outline-hidden" title="Like"><svg class="left-1.5 absolute" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32" fill="currentColor"><path d="M22.45,6a5.47,5.47,0,0,1,3.91,1.64,5.7,5.7,0,0,1,0,8L16,26.13,5.64,15.64a5.7,5.7,0,0,1,0-8,5.48,5.48,0,0,1,7.82,0L16,10.24l2.53-2.58A5.44,5.44,0,0,1,22.45,6m0-2a7.47,7.47,0,0,0-5.34,2.24L16,7.36,14.89,6.24a7.49,7.49,0,0,0-10.68,0,7.72,7.72,0,0,0,0,10.82L16,29,27.79,17.06a7.72,7.72,0,0,0,0-10.82A7.49,7.49,0,0,0,22.45,4Z"></path></svg>
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
<span class="ml-4 pl-0.5 ">like</span></button>
|
| 138 |
+
<button class="focus:outline-hidden flex items-center border-l px-1.5 py-1 text-gray-400 hover:bg-gray-50 focus:bg-gray-100 dark:hover:bg-gray-900 dark:focus:bg-gray-800" title="See users who liked this repository">0</button></div>
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
</h1>
|
| 144 |
+
<div class="mb-3 flex flex-wrap md:mb-4"><a class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" href="/models?pipeline_tag=text-ranking"><div class="tag tag-white "><div class="tag-ico -ml-2 tag-ico-red"><svg class="-mr-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M2 9h9V2H2zm2-5h5v3H4z" fill="currentColor"></path><path d="M2 19h9v-7H2zm2-5h5v3H4z" fill="currentColor"></path><path d="M2 29h9v-7H2zm2-5h5v3H4z" fill="currentColor"></path><path d="M27 9h-9l3.41-3.59L20 4l-6 6l6 6l1.41-1.41L18 11h9a1 1 0 0 1 1 1v12a1 1 0 0 1-1 1H15v2h12a3 3 0 0 0 3-3V12a3 3 0 0 0-3-3z" fill="currentColor"></path></svg></div>
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
<span>Text Ranking</span>
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
</div></a><a class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" href="/models?library=transformers"><div class="tag tag-white "><svg class="text-black inline-block text-sm" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" preserveAspectRatio="xMidYMid meet" width="1em" height="1em" viewBox="0 0 95 88"><path fill="#fff" d="M94.25 70.08a8.28 8.28 0 0 1-.43 6.46 10.57 10.57 0 0 1-3 3.6 25.18 25.18 0 0 1-5.7 3.2 65.74 65.74 0 0 1-7.56 2.65 46.67 46.67 0 0 1-11.42 1.68c-5.42.05-10.09-1.23-13.4-4.5a40.4 40.4 0 0 1-10.14.03c-3.34 3.25-7.99 4.52-13.39 4.47a46.82 46.82 0 0 1-11.43-1.68 66.37 66.37 0 0 1-7.55-2.65c-2.28-.98-4.17-2-5.68-3.2a10.5 10.5 0 0 1-3.02-3.6c-.99-2-1.18-4.3-.42-6.46a8.54 8.54 0 0 1-.33-5.63c.25-.95.66-1.83 1.18-2.61a8.67 8.67 0 0 1 2.1-8.47 8.23 8.23 0 0 1 2.82-2.07 41.75 41.75 0 1 1 81.3-.12 8.27 8.27 0 0 1 3.11 2.19 8.7 8.7 0 0 1 2.1 8.47c.52.78.93 1.66 1.18 2.61a8.61 8.61 0 0 1-.32 5.63Z"></path><path fill="#FFD21E" d="M47.21 76.5a34.75 34.75 0 1 0 0-69.5 34.75 34.75 0 0 0 0 69.5Z"></path><path fill="#FF9D0B" d="M81.96 41.75a34.75 34.75 0 1 0-69.5 0 34.75 34.75 0 0 0 69.5 0Zm-73.5 0a38.75 38.75 0 1 1 77.5 0 38.75 38.75 0 0 1-77.5 0Z"></path><path fill="#3A3B45" d="M58.5 32.3c1.28.44 1.78 3.06 3.07 2.38a5 5 0 1 0-6.76-2.07c.61 1.15 2.55-.72 3.7-.32ZM34.95 32.3c-1.28.44-1.79 3.06-3.07 2.38a5 5 0 1 1 6.76-2.07c-.61 1.15-2.56-.72-3.7-.32Z"></path><path fill="#FF323D" d="M46.96 56.29c9.83 0 13-8.76 13-13.26 0-2.34-1.57-1.6-4.09-.36-2.33 1.15-5.46 2.74-8.9 2.74-7.19 0-13-6.88-13-2.38s3.16 13.26 13 13.26Z"></path><path fill="#3A3B45" fill-rule="evenodd" d="M39.43 54a8.7 8.7 0 0 1 5.3-4.49c.4-.12.81.57 1.24 1.28.4.68.82 1.37 1.24 1.37.45 0 .9-.68 1.33-1.35.45-.7.89-1.38 1.32-1.25a8.61 8.61 0 0 1 5 4.17c3.73-2.94 5.1-7.74 5.1-10.7 0-2.34-1.57-1.6-4.09-.36l-.14.07c-2.31 1.15-5.39 2.67-8.77 2.67s-6.45-1.52-8.77-2.67c-2.6-1.29-4.23-2.1-4.23.29 0 3.05 1.46 8.06 5.47 10.97Z" clip-rule="evenodd"></path><path fill="#FF9D0B" d="M70.71 37a3.25 3.25 0 1 0 0-6.5 3.25 3.25 0 0 0 0 6.5ZM24.21 37a3.25 3.25 0 1 0 0-6.5 3.25 3.25 0 0 0 0 6.5ZM17.52 48c-1.62 0-3.06.66-4.07 1.87a5.97 5.97 0 0 0-1.33 3.76 7.1 7.1 0 0 0-1.94-.3c-1.55 0-2.95.59-3.94 1.66a5.8 5.8 0 0 0-.8 7 5.3 5.3 0 0 0-1.79 2.82c-.24.9-.48 2.8.8 4.74a5.22 5.22 0 0 0-.37 5.02c1.02 2.32 3.57 4.14 8.52 6.1 3.07 1.22 5.89 2 5.91 2.01a44.33 44.33 0 0 0 10.93 1.6c5.86 0 10.05-1.8 12.46-5.34 3.88-5.69 3.33-10.9-1.7-15.92-2.77-2.78-4.62-6.87-5-7.77-.78-2.66-2.84-5.62-6.25-5.62a5.7 5.7 0 0 0-4.6 2.46c-1-1.26-1.98-2.25-2.86-2.82A7.4 7.4 0 0 0 17.52 48Zm0 4c.51 0 1.14.22 1.82.65 2.14 1.36 6.25 8.43 7.76 11.18.5.92 1.37 1.31 2.14 1.31 1.55 0 2.75-1.53.15-3.48-3.92-2.93-2.55-7.72-.68-8.01.08-.02.17-.02.24-.02 1.7 0 2.45 2.93 2.45 2.93s2.2 5.52 5.98 9.3c3.77 3.77 3.97 6.8 1.22 10.83-1.88 2.75-5.47 3.58-9.16 3.58-3.81 0-7.73-.9-9.92-1.46-.11-.03-13.45-3.8-11.76-7 .28-.54.75-.76 1.34-.76 2.38 0 6.7 3.54 8.57 3.54.41 0 .7-.17.83-.6.79-2.85-12.06-4.05-10.98-8.17.2-.73.71-1.02 1.44-1.02 3.14 0 10.2 5.53 11.68 5.53.11 0 .2-.03.24-.1.74-1.2.33-2.04-4.9-5.2-5.21-3.16-8.88-5.06-6.8-7.33.24-.26.58-.38 1-.38 3.17 0 10.66 6.82 10.66 6.82s2.02 2.1 3.25 2.1c.28 0 .52-.1.68-.38.86-1.46-8.06-8.22-8.56-11.01-.34-1.9.24-2.85 1.31-2.85Z"></path><path fill="#FFD21E" d="M38.6 76.69c2.75-4.04 2.55-7.07-1.22-10.84-3.78-3.77-5.98-9.3-5.98-9.3s-.82-3.2-2.69-2.9c-1.87.3-3.24 5.08.68 8.01 3.91 2.93-.78 4.92-2.29 2.17-1.5-2.75-5.62-9.82-7.76-11.18-2.13-1.35-3.63-.6-3.13 2.2.5 2.79 9.43 9.55 8.56 11-.87 1.47-3.93-1.71-3.93-1.71s-9.57-8.71-11.66-6.44c-2.08 2.27 1.59 4.17 6.8 7.33 5.23 3.16 5.64 4 4.9 5.2-.75 1.2-12.28-8.53-13.36-4.4-1.08 4.11 11.77 5.3 10.98 8.15-.8 2.85-9.06-5.38-10.74-2.18-1.7 3.21 11.65 6.98 11.76 7.01 4.3 1.12 15.25 3.49 19.08-2.12Z"></path><path fill="#FF9D0B" d="M77.4 48c1.62 0 3.07.66 4.07 1.87a5.97 5.97 0 0 1 1.33 3.76 7.1 7.1 0 0 1 1.95-.3c1.55 0 2.95.59 3.94 1.66a5.8 5.8 0 0 1 .8 7 5.3 5.3 0 0 1 1.78 2.82c.24.9.48 2.8-.8 4.74a5.22 5.22 0 0 1 .37 5.02c-1.02 2.32-3.57 4.14-8.51 6.1-3.08 1.22-5.9 2-5.92 2.01a44.33 44.33 0 0 1-10.93 1.6c-5.86 0-10.05-1.8-12.46-5.34-3.88-5.69-3.33-10.9 1.7-15.92 2.78-2.78 4.63-6.87 5.01-7.77.78-2.66 2.83-5.62 6.24-5.62a5.7 5.7 0 0 1 4.6 2.46c1-1.26 1.98-2.25 2.87-2.82A7.4 7.4 0 0 1 77.4 48Zm0 4c-.51 0-1.13.22-1.82.65-2.13 1.36-6.25 8.43-7.76 11.18a2.43 2.43 0 0 1-2.14 1.31c-1.54 0-2.75-1.53-.14-3.48 3.91-2.93 2.54-7.72.67-8.01a1.54 1.54 0 0 0-.24-.02c-1.7 0-2.45 2.93-2.45 2.93s-2.2 5.52-5.97 9.3c-3.78 3.77-3.98 6.8-1.22 10.83 1.87 2.75 5.47 3.58 9.15 3.58 3.82 0 7.73-.9 9.93-1.46.1-.03 13.45-3.8 11.76-7-.29-.54-.75-.76-1.34-.76-2.38 0-6.71 3.54-8.57 3.54-.42 0-.71-.17-.83-.6-.8-2.85 12.05-4.05 10.97-8.17-.19-.73-.7-1.02-1.44-1.02-3.14 0-10.2 5.53-11.68 5.53-.1 0-.19-.03-.23-.1-.74-1.2-.34-2.04 4.88-5.2 5.23-3.16 8.9-5.06 6.8-7.33-.23-.26-.57-.38-.98-.38-3.18 0-10.67 6.82-10.67 6.82s-2.02 2.1-3.24 2.1a.74.74 0 0 1-.68-.38c-.87-1.46 8.05-8.22 8.55-11.01.34-1.9-.24-2.85-1.31-2.85Z"></path><path fill="#FFD21E" d="M56.33 76.69c-2.75-4.04-2.56-7.07 1.22-10.84 3.77-3.77 5.97-9.3 5.97-9.3s.82-3.2 2.7-2.9c1.86.3 3.23 5.08-.68 8.01-3.92 2.93.78 4.92 2.28 2.17 1.51-2.75 5.63-9.82 7.76-11.18 2.13-1.35 3.64-.6 3.13 2.2-.5 2.79-9.42 9.55-8.55 11 .86 1.47 3.92-1.71 3.92-1.71s9.58-8.71 11.66-6.44c2.08 2.27-1.58 4.17-6.8 7.33-5.23 3.16-5.63 4-4.9 5.2.75 1.2 12.28-8.53 13.36-4.4 1.08 4.11-11.76 5.3-10.97 8.15.8 2.85 9.05-5.38 10.74-2.18 1.69 3.21-11.65 6.98-11.76 7.01-4.31 1.12-15.26 3.49-19.08-2.12Z"></path></svg>
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
<span>Transformers</span>
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
</div></a><a class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" href="/models?library=safetensors"><div class="tag tag-white "><svg class="text-black inline-block text-sm" viewBox="0 0 57 44" fill="none" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet"><path d="M36.816 20.1474L54.9918 27.4409C55.5142 27.6506 55.9623 28.0112 56.2788 28.4766C56.5954 28.9421 56.7661 29.4913 56.7691 30.0542C56.7722 30.6171 56.6074 31.1682 56.2959 31.637C55.9844 32.1059 55.5402 32.4713 55.0201 32.6866L29.953 43.0646C29.2593 43.3518 28.4799 43.3518 27.7862 43.0646L2.71624 32.6894C2.19613 32.4741 1.75197 32.1087 1.44044 31.6398C1.12892 31.171 0.964165 30.62 0.967204 30.057C0.970244 29.4941 1.14094 28.9449 1.45751 28.4794C1.77408 28.014 2.22216 27.6534 2.74456 27.4437L21.2404 20.0227C22.2997 19.5979 25.6477 20.8441 28.8682 20.8555C32.3096 20.8668 35.6292 19.6715 36.816 20.1474ZM11.3042 30.1119L28.8682 37.3828L46.435 30.1119L28.8682 23.0619L11.3042 30.1119ZM29.9247 0.388251L54.9918 10.4462C55.5142 10.6559 55.9623 11.0165 56.2788 11.482C56.5954 11.9474 56.7661 12.4967 56.7691 13.0596C56.7722 13.6225 56.6074 14.1735 56.2959 14.6424C55.9844 15.1112 55.5402 15.4766 55.0201 15.6919L29.953 26.07C29.2593 26.3572 28.4799 26.3572 27.7862 26.07L2.71624 15.6948C2.19613 15.4795 1.75197 15.1141 1.44044 14.6452C1.12892 14.1763 0.964165 13.6253 0.967204 13.0624C0.970244 12.4995 1.14094 11.9503 1.45751 11.4848C1.77408 11.0193 2.22216 10.6588 2.74456 10.4491L27.8117 0.388251C28.4896 0.1157 29.2467 0.1157 29.9247 0.388251ZM11.3042 13.1172L28.8682 20.3881L46.435 13.1172L28.8682 6.06729L11.3042 13.1172Z" fill="currentColor"></path></svg>
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
<span>Safetensors</span>
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
</div></a>
|
| 166 |
+
<button class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" type="button"><div class="tag tag-white ">
|
| 167 |
+
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="text-green-600/80" preserveAspectRatio="xMidYMid meet" width="1em" height="1em" viewBox="0 0 10 10"><path fill-rule="evenodd" clip-rule="evenodd" d="M0.625 5C0.625 6.16032 1.08594 7.27312 1.90641 8.09359C2.72688 8.91406 3.83968 9.375 5 9.375C6.16032 9.375 7.27312 8.91406 8.09359 8.09359C8.91406 7.27312 9.375 6.16032 9.375 5C9.375 3.83968 8.91406 2.72688 8.09359 1.90641C7.27312 1.08594 6.16032 0.625 5 0.625C3.83968 0.625 2.72688 1.08594 1.90641 1.90641C1.08594 2.72688 0.625 3.83968 0.625 5ZM7.64365 7.48027C7.61734 7.50832 7.59054 7.53598 7.56326 7.56326C7.13828 7.98824 6.61864 8.2968 6.0539 8.46842C6.29802 8.11949 6.49498 7.64804 6.63475 7.09483C7.00845 7.18834 7.35014 7.3187 7.64365 7.48027ZM8.10076 6.87776C8.37677 6.42196 8.55005 5.90894 8.60556 5.37499H6.86808C6.85542 5.71597 6.82551 6.04557 6.77971 6.35841C7.25309 6.47355 7.68808 6.6414 8.062 6.85549C8.07497 6.86283 8.08789 6.87025 8.10076 6.87776ZM6.03795 6.22536C6.07708 5.95737 6.1044 5.67232 6.11705 5.37499H3.88295C3.89666 5.69742 3.92764 6.00542 3.9722 6.29287C4.37075 6.21726 4.79213 6.17749 5.224 6.17749C5.50054 6.17749 5.77294 6.19376 6.03795 6.22536ZM4.1261 7.02673C4.34894 7.84835 4.68681 8.375 5 8.375C5.32122 8.375 5.66839 7.82101 5.8908 6.963C5.67389 6.93928 5.45082 6.92699 5.224 6.92699C4.84316 6.92699 4.47332 6.96176 4.1261 7.02673ZM3.39783 7.21853C3.53498 7.71842 3.72038 8.14579 3.9461 8.46842C3.42141 8.30898 2.93566 8.03132 2.52857 7.65192C2.77253 7.48017 3.06711 7.33382 3.39783 7.21853ZM3.23916 6.48077C3.18263 6.13193 3.14625 5.76074 3.13192 5.37499H1.39444C1.4585 5.99112 1.67936 6.57938 2.03393 7.08403C2.3706 6.83531 2.78055 6.63162 3.23916 6.48077ZM1.39444 4.62499H3.13192C3.14615 4.24204 3.18211 3.87344 3.23794 3.52681C2.77814 3.37545 2.36731 3.17096 2.03024 2.92123C1.67783 3.42469 1.45828 4.011 1.39444 4.62499ZM2.5237 2.35262C2.76812 2.52552 3.06373 2.67281 3.39584 2.78875C3.53318 2.28573 3.71928 1.85578 3.9461 1.53158C3.41932 1.69166 2.93178 1.97089 2.5237 2.35262ZM3.97101 3.71489C3.92709 4.00012 3.89654 4.30547 3.88295 4.62499H6.11705C6.10453 4.33057 6.07761 4.04818 6.03909 3.78248C5.77372 3.81417 5.50093 3.83049 5.224 3.83049C4.79169 3.83049 4.3699 3.79065 3.97101 3.71489ZM5.8928 3.04476C5.67527 3.06863 5.45151 3.08099 5.224 3.08099C4.84241 3.08099 4.47186 3.04609 4.12405 2.98086C4.34686 2.1549 4.68584 1.625 5 1.625C5.32218 1.625 5.67048 2.18233 5.8928 3.04476ZM6.78083 3.6493C6.826 3.95984 6.85552 4.28682 6.86808 4.62499H8.60556C8.55029 4.09337 8.37827 3.58251 8.10436 3.1282C8.0903 3.1364 8.07618 3.14449 8.062 3.15249C7.68838 3.36641 7.25378 3.53417 6.78083 3.6493ZM7.64858 2.52499C7.35446 2.68754 7.0117 2.81868 6.63664 2.91268C6.49676 2.35623 6.29913 1.88209 6.0539 1.53158C6.61864 1.7032 7.13828 2.01176 7.56326 2.43674C7.59224 2.46572 7.62068 2.49514 7.64858 2.52499Z" fill="currentColor"></path></svg>
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
<span>16 languages</span>
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
</div></button><a class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" href="/models?other=xlm-roberta"><div class="tag tag-white ">
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
<span>xlm-roberta</span>
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
</div></a><a class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" href="/models?other=feature-extraction"><div class="tag tag-white ">
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
<span>feature-extraction</span>
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
</div></a><a class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" href="/models?other=XProvence"><div class="tag tag-white ">
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
<span>XProvence</span>
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
</div></a><a class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" href="/models?other=RAG"><div class="tag tag-white ">
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
<span>RAG</span>
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
</div></a><a class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" href="/models?other=Multilingual"><div class="tag tag-white ">
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
<span>Multilingual</span>
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
</div></a><a class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" href="/models?other=Provence"><div class="tag tag-white ">
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
<span>Provence</span>
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
</div></a><a class="mb-1 mr-1 md:mb-1.5 md:mr-1.5 rounded-lg" href="/models?other=custom_code"><div class="tag tag-white ">
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
<span>custom_code</span>
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
</div></a><div class="relative inline-block ">
|
| 224 |
+
<button class="group mr-1 mb-1 md:mr-1.5 md:mb-1.5 rounded-full rounded-br-none " type="button">
|
| 225 |
+
<div slot="button"><div class="tag rounded-full tag-white relative rounded-br-none pr-2.5">
|
| 226 |
+
<svg class="text-xs text-gray-900" width="1em" height="1em" viewBox="0 0 10 10" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1.46009 5.0945V6.88125C1.46009 7.25201 1.75937 7.55129 2.13012 7.55129C2.50087 7.55129 2.80016 7.25201 2.80016 6.88125V5.0945C2.80016 4.72375 2.50087 4.42446 2.13012 4.42446C1.75937 4.42446 1.46009 4.72375 1.46009 5.0945ZM4.14022 5.0945V6.88125C4.14022 7.25201 4.4395 7.55129 4.81026 7.55129C5.18101 7.55129 5.48029 7.25201 5.48029 6.88125V5.0945C5.48029 4.72375 5.18101 4.42446 4.81026 4.42446C4.4395 4.42446 4.14022 4.72375 4.14022 5.0945ZM1.23674 9.78473H8.38377C8.75452 9.78473 9.0538 9.48545 9.0538 9.1147C9.0538 8.74395 8.75452 8.44466 8.38377 8.44466H1.23674C0.865993 8.44466 0.566711 8.74395 0.566711 9.1147C0.566711 9.48545 0.865993 9.78473 1.23674 9.78473ZM6.82036 5.0945V6.88125C6.82036 7.25201 7.11964 7.55129 7.49039 7.55129C7.86114 7.55129 8.16042 7.25201 8.16042 6.88125V5.0945C8.16042 4.72375 7.86114 4.42446 7.49039 4.42446C7.11964 4.42446 6.82036 4.72375 6.82036 5.0945ZM4.39484 0.623142L0.865993 2.48137C0.682851 2.57517 0.566711 2.76725 0.566711 2.97273C0.566711 3.28094 0.816857 3.53109 1.12507 3.53109H8.49991C8.80365 3.53109 9.0538 3.28094 9.0538 2.97273C9.0538 2.76725 8.93766 2.57517 8.75452 2.48137L5.22568 0.623142C4.9666 0.484669 4.65391 0.484669 4.39484 0.623142V0.623142Z" fill="currentColor"></path></svg>
|
| 227 |
+
|
| 228 |
+
<span class="-mr-1 text-gray-400">License:</span>
|
| 229 |
+
|
| 230 |
+
<span>cc-by-nc-nd-4.0</span>
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
<div class="border-br-gray-200 absolute bottom-0.5 right-0.5 h-1 w-1 border-[3px] border-l-transparent border-t-transparent border-b-gray-200 border-r-gray-200 group-hover:border-b-gray-400 group-hover:border-r-gray-400 dark:border-b-gray-700 dark:border-r-gray-700 group-hover:dark:border-b-gray-400 group-hover:dark:border-r-gray-400"></div></div></div>
|
| 234 |
+
</button>
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
</div></div>
|
| 238 |
+
|
| 239 |
+
<div class="flex flex-col-reverse lg:flex-row lg:items-center lg:justify-between"><div class="-mb-px flex h-12 items-center overflow-x-auto overflow-y-hidden ">
|
| 240 |
+
<a class="tab-alternate" href="/youssef101/xprovence"><svg class="mr-1.5 text-gray-400 flex-none" style="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg>
|
| 241 |
+
Model card
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
</a><a class="tab-alternate active" href="/youssef101/xprovence/tree/main"><svg class="mr-1.5 text-gray-400 flex-none" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-tertiary" d="M21 19h-8a1 1 0 0 1 0-2h8a1 1 0 0 1 0 2zm0-4h-8a1 1 0 0 1 0-2h8a1 1 0 0 1 0 2zm0-8h-8a1 1 0 0 1 0-2h8a1 1 0 0 1 0 2zm0 4h-8a1 1 0 0 1 0-2h8a1 1 0 0 1 0 2z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M9 19a1 1 0 0 1-1-1V6a1 1 0 0 1 2 0v12a1 1 0 0 1-1 1zm-6-4.333a1 1 0 0 1-.64-1.769L3.438 12l-1.078-.898a1 1 0 0 1 1.28-1.538l2 1.667a1 1 0 0 1 0 1.538l-2 1.667a.999.999 0 0 1-.64.231z" fill="currentColor"></path></svg>
|
| 246 |
+
<span class="xl:hidden">Files</span>
|
| 247 |
+
<span class="hidden xl:inline">Files and versions</span>
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
<span class="inline-block "><span class="contents"><div slot="anchor" class="shadow-purple-500/10 ml-2 inline-flex -translate-y-px items-center gap-0.5 rounded-md border bg-white px-1 py-0.5 align-middle text-xs font-semibold leading-none text-gray-800 shadow-sm dark:border-gray-700 dark:bg-gradient-to-b dark:from-gray-925 dark:to-gray-925 dark:text-gray-300"><svg class="size-3 " xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 12 12"><path fill-rule="evenodd" clip-rule="evenodd" d="M6.14 3.64 5.1 4.92 2.98 2.28h2.06l1.1 1.36Zm0 4.72-1.1 1.36H2.98l2.13-2.64 1.03 1.28Zm4.9 1.36L8.03 6l3-3.72H8.96L5.97 6l3 3.72h2.06Z" fill="#7875FF"></path><path d="M4.24 6 2.6 8.03.97 6 2.6 3.97 4.24 6Z" fill="#FF7F41" opacity="1"></path></svg>
|
| 253 |
+
<span>xet</span>
|
| 254 |
+
</div></span>
|
| 255 |
+
</span>
|
| 256 |
+
</a><a class="tab-alternate" href="/youssef101/xprovence/discussions"><svg class="mr-1.5 text-gray-400 flex-none" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M20.6081 3C21.7684 3 22.8053 3.49196 23.5284 4.38415C23.9756 4.93678 24.4428 5.82749 24.4808 7.16133C24.9674 7.01707 25.4353 6.93643 25.8725 6.93643C26.9833 6.93643 27.9865 7.37587 28.696 8.17411C29.6075 9.19872 30.0124 10.4579 29.8361 11.7177C29.7523 12.3177 29.5581 12.8555 29.2678 13.3534C29.8798 13.8646 30.3306 14.5763 30.5485 15.4322C30.719 16.1032 30.8939 17.5006 29.9808 18.9403C30.0389 19.0342 30.0934 19.1319 30.1442 19.2318C30.6932 20.3074 30.7283 21.5229 30.2439 22.6548C29.5093 24.3704 27.6841 25.7219 24.1397 27.1727C21.9347 28.0753 19.9174 28.6523 19.8994 28.6575C16.9842 29.4379 14.3477 29.8345 12.0653 29.8345C7.87017 29.8345 4.8668 28.508 3.13831 25.8921C0.356375 21.6797 0.754104 17.8269 4.35369 14.1131C6.34591 12.058 7.67023 9.02782 7.94613 8.36275C8.50224 6.39343 9.97271 4.20438 12.4172 4.20438H12.4179C12.6236 4.20438 12.8314 4.2214 13.0364 4.25468C14.107 4.42854 15.0428 5.06476 15.7115 6.02205C16.4331 5.09583 17.134 4.359 17.7682 3.94323C18.7242 3.31737 19.6794 3 20.6081 3ZM20.6081 5.95917C20.2427 5.95917 19.7963 6.1197 19.3039 6.44225C17.7754 7.44319 14.8258 12.6772 13.7458 14.7131C13.3839 15.3952 12.7655 15.6837 12.2086 15.6837C11.1036 15.6837 10.2408 14.5497 12.1076 13.1085C14.9146 10.9402 13.9299 7.39584 12.5898 7.1776C12.5311 7.16799 12.4731 7.16355 12.4172 7.16355C11.1989 7.16355 10.6615 9.33114 10.6615 9.33114C10.6615 9.33114 9.0863 13.4148 6.38031 16.206C3.67434 18.998 3.5346 21.2388 5.50675 24.2246C6.85185 26.2606 9.42666 26.8753 12.0653 26.8753C14.8021 26.8753 17.6077 26.2139 19.1799 25.793C19.2574 25.7723 28.8193 22.984 27.6081 20.6107C27.4046 20.212 27.0693 20.0522 26.6471 20.0522C24.9416 20.0522 21.8393 22.6726 20.5057 22.6726C20.2076 22.6726 19.9976 22.5416 19.9116 22.222C19.3433 20.1173 28.552 19.2325 27.7758 16.1839C27.639 15.6445 27.2677 15.4256 26.746 15.4263C24.4923 15.4263 19.4358 19.5181 18.3759 19.5181C18.2949 19.5181 18.2368 19.4937 18.2053 19.4419C17.6743 18.557 17.9653 17.9394 21.7082 15.6009C25.4511 13.2617 28.0783 11.8545 26.5841 10.1752C26.4121 9.98141 26.1684 9.8956 25.8725 9.8956C23.6001 9.89634 18.2311 14.9403 18.2311 14.9403C18.2311 14.9403 16.7821 16.496 15.9057 16.496C15.7043 16.496 15.533 16.4139 15.4169 16.2112C14.7956 15.1296 21.1879 10.1286 21.5484 8.06535C21.7928 6.66715 21.3771 5.95917 20.6081 5.95917Z" fill="#FF9D00"></path><path d="M5.50686 24.2246C3.53472 21.2387 3.67446 18.9979 6.38043 16.206C9.08641 13.4147 10.6615 9.33111 10.6615 9.33111C10.6615 9.33111 11.2499 6.95933 12.59 7.17757C13.93 7.39581 14.9139 10.9401 12.1069 13.1084C9.29997 15.276 12.6659 16.7489 13.7459 14.713C14.8258 12.6772 17.7747 7.44316 19.304 6.44221C20.8326 5.44128 21.9089 6.00204 21.5484 8.06532C21.188 10.1286 14.795 15.1295 15.4171 16.2118C16.0391 17.2934 18.2312 14.9402 18.2312 14.9402C18.2312 14.9402 25.0907 8.49588 26.5842 10.1752C28.0776 11.8545 25.4512 13.2616 21.7082 15.6008C17.9646 17.9393 17.6744 18.557 18.2054 19.4418C18.7372 20.3266 26.9998 13.1351 27.7759 16.1838C28.5513 19.2324 19.3434 20.1173 19.9117 22.2219C20.48 24.3274 26.3979 18.2382 27.6082 20.6107C28.8193 22.9839 19.2574 25.7722 19.18 25.7929C16.0914 26.62 8.24723 28.3726 5.50686 24.2246Z" fill="#FFD21E"></path></svg>
|
| 257 |
+
Community
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
</a></div>
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
<div class="relative mb-1.5 flex flex-wrap gap-1.5 sm:flex-nowrap lg:mb-0"><div class="order-last sm:order-first"><div class="relative ">
|
| 267 |
+
<button class="btn px-1.5 py-1.5 " type="button">
|
| 268 |
+
|
| 269 |
+
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="p-0.5" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><circle cx="16" cy="7" r="3" fill="currentColor"></circle><circle cx="16" cy="16" r="3" fill="currentColor"></circle><circle cx="16" cy="25" r="3" fill="currentColor"></circle></svg>
|
| 270 |
+
|
| 271 |
+
</button>
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
</div></div>
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
<div class="flex-none w-full sm:w-auto"><div class="relative ">
|
| 290 |
+
<button class="text-sm btn btn w-full cursor-pointer text-sm" type="button">
|
| 291 |
+
<svg class="mr-1.5 " xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M12.1 2a9.8 9.8 0 0 0-5.4 1.6l6.4 6.4a2.1 2.1 0 0 1 .2 3a2.1 2.1 0 0 1-3-.2L3.7 6.4A9.84 9.84 0 0 0 2 12.1a10.14 10.14 0 0 0 10.1 10.1a10.9 10.9 0 0 0 2.6-.3l6.7 6.7a5 5 0 0 0 7.1-7.1l-6.7-6.7a10.9 10.9 0 0 0 .3-2.6A10 10 0 0 0 12.1 2zm8 10.1a7.61 7.61 0 0 1-.3 2.1l-.3 1.1l.8.8l6.7 6.7a2.88 2.88 0 0 1 .9 2.1A2.72 2.72 0 0 1 27 27a2.9 2.9 0 0 1-4.2 0l-6.7-6.7l-.8-.8l-1.1.3a7.61 7.61 0 0 1-2.1.3a8.27 8.27 0 0 1-5.7-2.3A7.63 7.63 0 0 1 4 12.1a8.33 8.33 0 0 1 .3-2.2l4.4 4.4a4.14 4.14 0 0 0 5.9.2a4.14 4.14 0 0 0-.2-5.9L10 4.2a6.45 6.45 0 0 1 2-.3a8.27 8.27 0 0 1 5.7 2.3a8.49 8.49 0 0 1 2.4 5.9z" fill="currentColor"></path></svg>
|
| 292 |
+
Train
|
| 293 |
+
<svg class="-mr-1 text-gray-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path d="M16.293 9.293L12 13.586L7.707 9.293l-1.414 1.414L12 16.414l5.707-5.707z" fill="currentColor"></path></svg></button>
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
</div>
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
</div>
|
| 303 |
+
<div class="flex-none w-full sm:w-auto"><div class="relative ">
|
| 304 |
+
<button class="text-sm btn btn w-full cursor-pointer text-sm" type="button">
|
| 305 |
+
<svg class="mr-1.5 " xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><rect x="6.34" y="19" width="11.31" height="2" transform="translate(-10.63 14.34) rotate(-45)"></rect><path d="M17,30a1,1,0,0,1-.37-.07,1,1,0,0,1-.62-.79l-1-7,2-.28.75,5.27L21,24.52V17a1,1,0,0,1,.29-.71l4.07-4.07A8.94,8.94,0,0,0,28,5.86V4H26.14a8.94,8.94,0,0,0-6.36,2.64l-4.07,4.07A1,1,0,0,1,15,11H7.48L4.87,14.26l5.27.75-.28,2-7-1a1,1,0,0,1-.79-.62,1,1,0,0,1,.15-1l4-5A1,1,0,0,1,7,9h7.59l3.77-3.78A10.92,10.92,0,0,1,26.14,2H28a2,2,0,0,1,2,2V5.86a10.92,10.92,0,0,1-3.22,7.78L23,17.41V25a1,1,0,0,1-.38.78l-5,4A1,1,0,0,1,17,30Z"></path></svg>
|
| 306 |
+
Deploy
|
| 307 |
+
<svg class="-mr-1 text-gray-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path d="M16.293 9.293L12 13.586L7.707 9.293l-1.414 1.414L12 16.414l5.707-5.707z" fill="currentColor"></path></svg></button>
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
</div>
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
</div>
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
<div class="relative flex-auto sm:flex-none">
|
| 333 |
+
<button class="from-gray-800! to-black! text-white! gap-1! border-gray-800! dark:border-gray-900! btn w-full cursor-pointer text-sm" type="button">
|
| 334 |
+
<svg class="mr-1.5 mr-0.5! " xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path fill="currentColor" d="M28 4H4a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h8v4H8v2h16v-2h-4v-4h8a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2ZM18 28h-4v-4h4Zm10-6H4V6h24Z"></path></svg>
|
| 335 |
+
Use this model
|
| 336 |
+
<svg class="-mr-1 text-gray-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path d="M16.293 9.293L12 13.586L7.707 9.293l-1.414 1.414L12 16.414l5.707-5.707z" fill="currentColor"></path></svg></button>
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
</div>
|
| 340 |
+
|
| 341 |
+
</div>
|
| 342 |
+
</div></div></header>
|
| 343 |
+
</div>
|
| 344 |
+
|
| 345 |
+
<div class="container relative flex flex-col md:grid md:space-y-0 w-full md:grid-cols-12 space-y-4 md:gap-6 mb-16"><section class="pt-8 border-gray-100 col-span-full"><div class="SVELTE_HYDRATER contents" data-target="ViewerHeader" data-props="{"context":{"repo":{"name":"youssef101/xprovence","type":"model"},"rev":"main","path":"README.md","subpaths":[{"dir":"README.md"}]},"refs":{"branches":[{"name":"main","ref":"refs/heads/main","targetCommit":"49d7467109c3baedb5b4cee7ad6e451471e23547"}],"tags":[],"converts":[]},"view":"blob"}"><header class="flex flex-wrap items-center justify-start pb-2 md:justify-end lg:flex-nowrap"><div class="grow max-md:flex max-md:w-full max-md:items-start max-md:justify-between"><div class="relative mr-4 flex min-w-0 basis-auto flex-wrap items-center md:grow md:basis-full lg:basis-auto lg:flex-nowrap"><div class="relative mr-3 mb-2">
|
| 346 |
+
<button class="text-sm md:text-base btn w-full cursor-pointer text-sm" type="button">
|
| 347 |
+
<svg class="mr-1.5 text-gray-700 dark:text-gray-400" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24" style="transform: rotate(360deg);"><path d="M13 14c-3.36 0-4.46 1.35-4.82 2.24C9.25 16.7 10 17.76 10 19a3 3 0 0 1-3 3a3 3 0 0 1-3-3c0-1.31.83-2.42 2-2.83V7.83A2.99 2.99 0 0 1 4 5a3 3 0 0 1 3-3a3 3 0 0 1 3 3c0 1.31-.83 2.42-2 2.83v5.29c.88-.65 2.16-1.12 4-1.12c2.67 0 3.56-1.34 3.85-2.23A3.006 3.006 0 0 1 14 7a3 3 0 0 1 3-3a3 3 0 0 1 3 3c0 1.34-.88 2.5-2.09 2.86C17.65 11.29 16.68 14 13 14m-6 4a1 1 0 0 0-1 1a1 1 0 0 0 1 1a1 1 0 0 0 1-1a1 1 0 0 0-1-1M7 4a1 1 0 0 0-1 1a1 1 0 0 0 1 1a1 1 0 0 0 1-1a1 1 0 0 0-1-1m10 2a1 1 0 0 0-1 1a1 1 0 0 0 1 1a1 1 0 0 0 1-1a1 1 0 0 0-1-1z" fill="currentColor"></path></svg>
|
| 348 |
+
main
|
| 349 |
+
<svg class="-mr-1 text-gray-500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path d="M16.293 9.293L12 13.586L7.707 9.293l-1.414 1.414L12 16.414l5.707-5.707z" fill="currentColor"></path></svg></button>
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
</div>
|
| 353 |
+
<div class="relative mb-2 flex flex-wrap items-center"><a class="truncate text-gray-800 hover:underline" href="/youssef101/xprovence/tree/main">xprovence</a>
|
| 354 |
+
<span class="mx-1 text-gray-300">/</span>
|
| 355 |
+
<span class="dark:text-gray-300">README.md</span>
|
| 356 |
+
<button class="text-xs ml-2 focus:outline-hidden inline-flex cursor-pointer items-center text-sm mx-0.5 text-gray-600 " title="Copy path" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
|
| 357 |
+
</button></div></div>
|
| 358 |
+
</div>
|
| 359 |
+
|
| 360 |
+
</header></div>
|
| 361 |
+
<div class="SVELTE_HYDRATER contents" data-target="LastCommit" data-props="{"commitLast":{"date":"2025-09-04T11:14:55.000Z","verified":"verified","subject":"Update README.md","authors":[{"_id":"631dea73b07bce5f2a7cfdc4","avatar":"/avatars/f504ed0767b1bfdfc550075e477657a8.svg","isHf":false,"user":"youssef101"}],"commit":{"id":"49d7467109c3baedb5b4cee7ad6e451471e23547","parentIds":["9e812c6ada28c98c0c0851a25c63aac9a14db594"]},"title":"Update README.md"},"repo":{"name":"youssef101/xprovence","type":"model"}}"><div class="from-gray-100-to-white bg-linear-to-t flex flex-wrap items-baseline gap-y-1 rounded-t-lg border border-b-0 px-3 py-2 dark:border-gray-800"><img class="mr-2.5 mt-0.5 h-4 w-4 self-center rounded-full" alt="youssef101's picture" src="/avatars/f504ed0767b1bfdfc550075e477657a8.svg">
|
| 362 |
+
<div class="mr-4 flex flex-none items-center truncate"><a class="hover:underline" href="/youssef101">youssef101
|
| 363 |
+
</a>
|
| 364 |
+
|
| 365 |
+
</div>
|
| 366 |
+
<div class="mr-4 truncate font-mono text-xs text-gray-500 hover:prose-a:underline sm:text-sm"><!-- HTML_TAG_START -->Update README.md<!-- HTML_TAG_END --></div>
|
| 367 |
+
<a class="rounded-sm border bg-gray-50 px-1.5 text-sm hover:underline dark:border-gray-800 dark:bg-gray-900" href="/youssef101/xprovence/commit/49d7467109c3baedb5b4cee7ad6e451471e23547">49d7467</a>
|
| 368 |
+
<span class="mx-2 text-green-500 dark:text-green-600 px-1.5 border-green-100 dark:border-green-800 rounded-full border text-xs uppercase" title="This commit is signed and the signature is verified">verified</span>
|
| 369 |
+
<time class="ml-auto hidden flex-none truncate pl-2 text-gray-500 dark:text-gray-400 lg:block" datetime="2025-09-04T11:14:55" title="Thu, 04 Sep 2025 11:14:55 GMT">1 minute ago</time></div></div>
|
| 370 |
+
<div class="relative flex flex-wrap items-center border px-3 py-1.5 text-sm text-gray-800 dark:border-gray-800 dark:bg-gray-900 "><div class="flex items-center gap-3 text-sm font-medium"><a class="rounded-md px-1.5 capitalize bg-gray-200 dark:bg-gray-800" href="/youssef101/xprovence/blob/main/README.md">preview</a>
|
| 371 |
+
<a class="rounded-md px-1.5 capitalize " href="/youssef101/xprovence/blob/main/README.md?code=true">code</a></div>
|
| 372 |
+
<div class="mx-4 text-gray-200">|</div>
|
| 373 |
+
<a class="my-1 mr-4 flex items-center hover:underline " href="/youssef101/xprovence/raw/main/README.md"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32" style="transform: rotate(360deg);"><path d="M31 16l-7 7l-1.41-1.41L28.17 16l-5.58-5.59L24 9l7 7z" fill="currentColor"></path><path d="M1 16l7-7l1.41 1.41L3.83 16l5.58 5.59L8 23l-7-7z" fill="currentColor"></path><path d="M12.419 25.484L17.639 6l1.932.518L14.35 26z" fill="currentColor"></path></svg>
|
| 374 |
+
raw
|
| 375 |
+
</a><div class="SVELTE_HYDRATER contents" data-target="CopyButton" data-props="{"value":"https://huggingface.co/youssef101/xprovence/resolve/main/README.md","style":"blank","label":"Copy download link","classNames":"my-1 mr-4 flex items-center no-underline hover:underline"}"><button class="my-1 mr-4 flex items-center no-underline hover:underline " title="Copy download link" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
|
| 376 |
+
<span class="ml-1.5 ">Copy download link</span></button></div><a class="my-1 mr-4 flex items-center hover:underline " href="/youssef101/xprovence/commits/main/README.md"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32" style="transform: rotate(360deg);"><path d="M16 4C9.383 4 4 9.383 4 16s5.383 12 12 12s12-5.383 12-12S22.617 4 16 4zm0 2c5.535 0 10 4.465 10 10s-4.465 10-10 10S6 21.535 6 16S10.465 6 16 6zm-1 2v9h7v-2h-5V8z" fill="currentColor"></path></svg>
|
| 377 |
+
history
|
| 378 |
+
</a><a class="my-1 mr-4 flex items-center hover:underline " href="/youssef101/xprovence/blame/main/README.md"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32" style="transform: rotate(360deg);"><path d="M16 2a14 14 0 1 0 14 14A14 14 0 0 0 16 2zm0 26a12 12 0 1 1 12-12a12 12 0 0 1-12 12z" fill="currentColor"></path><path d="M11.5 11a2.5 2.5 0 1 0 2.5 2.5a2.48 2.48 0 0 0-2.5-2.5z" fill="currentColor"></path><path d="M20.5 11a2.5 2.5 0 1 0 2.5 2.5a2.48 2.48 0 0 0-2.5-2.5z" fill="currentColor"></path></svg>
|
| 379 |
+
blame
|
| 380 |
+
</a><a class="my-1 mr-4 flex items-center hover:underline text-green-600 dark:text-green-500" href="/youssef101/xprovence/edit/main/README.md"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M2 26h28v2H2z" fill="currentColor"></path><path d="M25.4 9c.8-.8.8-2 0-2.8l-3.6-3.6c-.8-.8-2-.8-2.8 0l-15 15V24h6.4l15-15zm-5-5L24 7.6l-3 3L17.4 7l3-3zM6 22v-3.6l10-10l3.6 3.6l-10 10H6z" fill="currentColor"></path></svg>
|
| 381 |
+
contribute
|
| 382 |
+
</a><a class="my-1 mr-4 flex items-center hover:underline " href="/youssef101/xprovence/delete/main/README.md"><svg class="mr-1.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M12 12h2v12h-2z" fill="currentColor"></path><path d="M18 12h2v12h-2z" fill="currentColor"></path><path d="M4 6v2h2v20a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V8h2V6zm4 22V8h16v20z" fill="currentColor"></path><path d="M12 2h8v2h-8z" fill="currentColor"></path></svg>
|
| 383 |
+
delete
|
| 384 |
+
</a>
|
| 385 |
+
|
| 386 |
+
<div class="mr-4 flex items-center"><div class="SVELTE_HYDRATER contents" data-target="ScanStatusBadge" data-props="{"classNames":"mr-2","scanStatus":{"status":"queued","protectAiScan":{"status":"queued","message":"","reportLink":""},"avScan":{"status":"safe","version":"1.4.3/27751"},"pickleImportScan":{"status":"unscanned","pickleImports":[],"version":"0.0.0"},"jFrogScan":{"status":"queued","message":"","reportLink":"","reportLabel":""}},"repo":{"name":"youssef101/xprovence","type":"model"},"revision":"main","filePath":"README.md","openByDefault":false}"><div class="sm:relative mr-2"><button class="flex h-[1.125rem] select-none items-center gap-0.5 rounded border pl-0.5 pr-0.5 text-xs leading-tight text-gray-400 hover:cursor-pointer text-gray-400 hover:border-gray-200 hover:bg-gray-50 hover:text-gray-500 dark:border-gray-800 dark:hover:bg-gray-800 dark:hover:text-gray-200 "><svg class="flex-none" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 25 26" fill="none"><path fill="currentColor" fill-opacity=".67" d="M12.324 12.04a2.038 2.038 0 0 0-2.033 2.033c0 1.118.915 2.033 2.033 2.033a2.038 2.038 0 0 0 2.032-2.033 2.038 2.038 0 0 0-2.032-2.032Zm6.097 2.033a6.102 6.102 0 0 0-6.097-6.097 6.102 6.102 0 0 0-6.098 6.097 6.094 6.094 0 0 0 3.049 5.274l1.016-1.768c-1.21-.711-2.032-2.002-2.032-3.506a4.064 4.064 0 0 1 4.065-4.065 4.064 4.064 0 0 1 4.065 4.065c0 1.504-.823 2.795-2.033 3.506l1.017 1.768a6.094 6.094 0 0 0 3.048-5.274ZM12.324 3.911C6.714 3.91 2.16 8.463 2.16 14.073c0 3.76 2.043 7.033 5.071 8.79l1.016-1.757c-2.418-1.413-4.054-4.025-4.054-7.033a8.128 8.128 0 0 1 8.13-8.13 8.128 8.128 0 0 1 8.13 8.13c0 3.008-1.636 5.62-4.065 7.033l1.016 1.758c3.039-1.758 5.081-5.03 5.081-8.79 0-5.61-4.553-10.163-10.162-10.163Z"></path><path fill="url(#icon-scan-queued-a)" d="m4.197 9.392 8.113 4.683-2.508-8.939-5.605 4.256Z"></path><defs><linearGradient id="icon-scan-queued-a" x1="11.957" x2="6.965" y1="13.453" y2="7.399" gradientUnits="userSpaceOnUse"><stop stop-color="#AEB6C2"></stop><stop offset="1" stop-color="#D1D5DB" stop-opacity="0"></stop></linearGradient></defs></svg></button>
|
| 387 |
+
|
| 388 |
+
</div></div>
|
| 389 |
+
</div>
|
| 390 |
+
|
| 391 |
+
<div class="flex items-center gap-x-3 dark:text-gray-300 sm:ml-auto">
|
| 392 |
+
8.17 kB</div></div>
|
| 393 |
+
|
| 394 |
+
<div class="relative min-h-[100px] overflow-hidden rounded-b-lg border border-t-0 leading-tight dark:border-gray-800 dark:bg-gray-925">
|
| 395 |
+
|
| 396 |
+
<div class="py-4 px-4 sm:px-6 prose hf-sanitized hf-sanitized-hhMmL5QYtta3-WzhwEHZy"><div class="not-prose bg-linear-to-t -mx-6 -mt-4 mb-8 max-h-[300px] min-w-full overflow-auto border-b from-gray-50 px-6 pb-5 pt-4 font-mono text-xs transition-all dark:from-gray-900 dark:to-gray-950"><div class="mb-2 inline-block rounded-lg border px-2 py-1 font-mono text-xs leading-none">metadata</div>
|
| 397 |
+
<pre><!-- HTML_TAG_START --><span class="hljs-attr">library_name:</span> <span class="hljs-string">transformers</span>
|
| 398 |
+
<span class="hljs-attr">tags:</span>
|
| 399 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">XProvence</span>
|
| 400 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">RAG</span>
|
| 401 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">Multilingual</span>
|
| 402 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">Provence</span>
|
| 403 |
+
<span class="hljs-attr">license:</span> <span class="hljs-string">cc-by-nc-nd-4.0</span>
|
| 404 |
+
<span class="hljs-attr">language:</span>
|
| 405 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">ar</span>
|
| 406 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">en</span>
|
| 407 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">fr</span>
|
| 408 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">es</span>
|
| 409 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">bn</span>
|
| 410 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">fa</span>
|
| 411 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">fi</span>
|
| 412 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">hi</span>
|
| 413 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">id</span>
|
| 414 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">ja</span>
|
| 415 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">ko</span>
|
| 416 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">ru</span>
|
| 417 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">sw</span>
|
| 418 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">te</span>
|
| 419 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">th</span>
|
| 420 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">zh</span>
|
| 421 |
+
<span class="hljs-attr">base_model:</span>
|
| 422 |
+
<span class="hljs-bullet">-</span> <span class="hljs-string">BAAI/bge-reranker-v2-m3</span>
|
| 423 |
+
<span class="hljs-attr">pipeline_tag:</span> <span class="hljs-string">text-ranking</span>
|
| 424 |
+
<!-- HTML_TAG_END --></pre></div>
|
| 425 |
+
<!-- HTML_TAG_START --><h1 class="relative group flex items-center">
|
| 426 |
+
<a rel="nofollow" href="#model-card-for-xprovence-reranker" class="block pr-1.5 text-lg md:absolute md:p-1.5 md:opacity-0 md:group-hover:opacity-100 md:right-full" id="model-card-for-xprovence-reranker">
|
| 427 |
+
<span class="header-link"><svg viewBox="0 0 256 256" preserveAspectRatio="xMidYMid meet" height="1em" width="1em" role="img" aria-hidden="true" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" class="text-gray-500 hover:text-black dark:hover:text-gray-200 w-4"><path fill="currentColor" d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z"></path></svg></span>
|
| 428 |
+
</a>
|
| 429 |
+
<span>
|
| 430 |
+
Model Card for XProvence-reranker
|
| 431 |
+
</span>
|
| 432 |
+
</h1>
|
| 433 |
+
<p>XProvence is a Zero Cost <strong>context pruning model</strong> that seamlessly integrates with reranker for retrieval-augmented generation,
|
| 434 |
+
particularly <strong>optimized for question answering</strong>. Given a user question and a retrieved passage, XProvence <strong>removes sentences
|
| 435 |
+
from the passage that are not relevant to the user question</strong>. This <strong>speeds up generation</strong> and <strong>reduces context noise</strong>, in
|
| 436 |
+
a plug-and-play manner <strong>for any LLM</strong>. </p>
|
| 437 |
+
<p>XProvence extends Provence by supporting 16 languages natively. It also supports 100+ languages through cross lingual transfer, since
|
| 438 |
+
it is based on BGE-m3 which is pretrained on 100+ languages. </p>
|
| 439 |
+
<p><em>Developed by</em>: Naver Labs Europe<br><em>License</em>: XProvence is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 license [CC BY-NC-ND 4.0 license].
|
| 440 |
+
<a rel="nofollow" href="https://huggingface.co/naver/provence-reranker-debertav3-v1/blob/main/Provence_LICENSE.txt">License file</a></p>
|
| 441 |
+
<ul>
|
| 442 |
+
<li><em>Model</em>: <code>XProvence</code> </li>
|
| 443 |
+
<li><em>Backbone model</em>: <a rel="nofollow" href="https://huggingface.co/BAAI/bge-reranker-v2-m3">bge-reranker-v2-m3</a></li>
|
| 444 |
+
<li><em>Model size</em>: 568 million parameters </li>
|
| 445 |
+
<li><em>Context length</em>: 8192 tokens</li>
|
| 446 |
+
</ul>
|
| 447 |
+
<p>Training and evaluation code & data are available in the <a rel="nofollow" href="https://github.com/naver/bergen/tree/main/scripts/provence">Bergen repo</a></p>
|
| 448 |
+
<h2 class="relative group flex items-center">
|
| 449 |
+
<a rel="nofollow" href="#usage" class="block pr-1.5 text-lg md:absolute md:p-1.5 md:opacity-0 md:group-hover:opacity-100 md:right-full" id="usage">
|
| 450 |
+
<span class="header-link"><svg viewBox="0 0 256 256" preserveAspectRatio="xMidYMid meet" height="1em" width="1em" role="img" aria-hidden="true" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" class="text-gray-500 hover:text-black dark:hover:text-gray-200 w-4"><path fill="currentColor" d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z"></path></svg></span>
|
| 451 |
+
</a>
|
| 452 |
+
<span>
|
| 453 |
+
Usage
|
| 454 |
+
</span>
|
| 455 |
+
</h2>
|
| 456 |
+
<p>Provence uses <code>spacy</code>:</p>
|
| 457 |
+
<pre><code class="language-bash">pip install spacy
|
| 458 |
+
python -m spacy download xx_sent_ud_sm
|
| 459 |
+
</code></pre>
|
| 460 |
+
<p>Pruning a single context for a single question:</p>
|
| 461 |
+
<pre><code class="language-python"><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel
|
| 462 |
+
|
| 463 |
+
xprovence = AutoModel.from_pretrained(<span class="hljs-string">"naver/XProvence"</span>, trust_remote_code=<span class="hljs-literal">True</span>)
|
| 464 |
+
|
| 465 |
+
context = <span class="hljs-string">"Shepherd’s pie. History. In early cookery books, the dish was a means of using leftover roasted meat of any kind, and the pie dish was lined on the sides and bottom with mashed potato, as well as having a mashed potato crust on top. Variations and similar dishes. Other potato-topped pies include: The modern ”Cumberland pie” is a version with either beef or lamb and a layer of bread- crumbs and cheese on top. In medieval times, and modern-day Cumbria, the pastry crust had a filling of meat with fruits and spices.. In Quebec, a varia- tion on the cottage pie is called ”Paˆte ́ chinois”. It is made with ground beef on the bottom layer, canned corn in the middle, and mashed potato on top.. The ”shepherdess pie” is a vegetarian version made without meat, or a vegan version made without meat and dairy.. In the Netherlands, a very similar dish called ”philosopher’s stew” () often adds ingredients like beans, apples, prunes, or apple sauce.. In Brazil, a dish called in refers to the fact that a manioc puree hides a layer of sun-dried meat."</span>
|
| 466 |
+
question = <span class="hljs-string">'What goes on the bottom of Shepherd’s pie?'</span>
|
| 467 |
+
|
| 468 |
+
xprovence_output = xprovence.process(question, context)
|
| 469 |
+
<span class="hljs-comment"># print(f"XProvence Output: {xprovence_output}")</span>
|
| 470 |
+
<span class="hljs-comment"># XProvence Output: {'reranking_score': 3.022725, pruned_context': 'In early cookery books, the dish was a means of using leftover roasted meat of any kind, and the pie dish was lined on the sides and bottom with mashed potato, as well as having a mashed potato crust on top.']]</span>
|
| 471 |
+
</code></pre>
|
| 472 |
+
<p>You can also pass a list of questions and a list of lists of contexts (multiple contexts per question to be pruned) for batched processing.</p>
|
| 473 |
+
<p>Setting <code>always_select_title=True</code> will keep the first sentence "Shepherd’s pie". This is especially useful for Wikipedia articles where the title is often needed to understand the context.
|
| 474 |
+
More details on how the title is defined are given below.</p>
|
| 475 |
+
<pre><code class="language-python">xprovence_output = xprovence.process(question, context, always_select_title=<span class="hljs-literal">True</span>)
|
| 476 |
+
<span class="hljs-comment"># print(f"XProvence Output: {xprovence_output}")</span>
|
| 477 |
+
<span class="hljs-comment"># XProvence Output: {'reranking_score': 3.022725, pruned_context': 'Shepherd’s pie. In early cookery books, the dish was a means of using leftover roasted meat of any kind, and the pie dish was lined on the sides and bottom with mashed potato, as well as having a mashed potato crust on top.']]</span>
|
| 478 |
+
</code></pre>
|
| 479 |
+
<h2 class="relative group flex items-center">
|
| 480 |
+
<a rel="nofollow" href="#model-interface" class="block pr-1.5 text-lg md:absolute md:p-1.5 md:opacity-0 md:group-hover:opacity-100 md:right-full" id="model-interface">
|
| 481 |
+
<span class="header-link"><svg viewBox="0 0 256 256" preserveAspectRatio="xMidYMid meet" height="1em" width="1em" role="img" aria-hidden="true" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" class="text-gray-500 hover:text-black dark:hover:text-gray-200 w-4"><path fill="currentColor" d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z"></path></svg></span>
|
| 482 |
+
</a>
|
| 483 |
+
<span>
|
| 484 |
+
Model interface
|
| 485 |
+
</span>
|
| 486 |
+
</h2>
|
| 487 |
+
<p>Interface of the <code>process</code> function:</p>
|
| 488 |
+
<ul>
|
| 489 |
+
<li><code>question</code>: <code>Union[List[str], str]</code>: an input question (str) or a list of input questions (for batched processing)</li>
|
| 490 |
+
<li><code>context</code>: <code>Union[List[List[str]], str]</code>: context(s) to be pruned. This can be either a single string (in case of a singe str question), or a list of lists contexts (a list of contexts per question), with <code>len(contexts)</code> equal to <code>len(questions)</code></li>
|
| 491 |
+
<li><code>title</code>: <code>Optional[Union[List[List[str]], str]]</code>, <em>default: “first_sentence”</em>: an optional argument for defining titles. If <code>title=first_sentence</code>, then the first sentence of each context is assumed to be the title. If <code>title=None</code>, then it is assumed that no titles are provided. Titles can be also passed as a list of lists of str, i.e. titles shaped the same way as contexts. Titles are only used if <code>always_select_title=True</code>.</li>
|
| 492 |
+
<li><code>threshold</code> <em>(float, $ \in [0, 1]$, default 0.3)</em>: which threshold to use for context pruning. We recommend 0.3 for more conservative pruning (no performance drop or lowest performance drops) and 0.7 for higher compression, but this value can be further tuned to meet the specific use case requirements.</li>
|
| 493 |
+
<li><code>always_select_title</code> <em>(bool, default: True)</em>: if True, the first sentence (title) will be included into the selection each time the model select a non-empty selection of sentences. This is important, e.g., for Wikipedia passages, to provide proper contextualization for the next sentences.</li>
|
| 494 |
+
<li><code>batch_size</code> (int, default: 32)</li>
|
| 495 |
+
<li><code>reorder</code> <em>(bool, default: False)</em>: if True, the provided contexts for each question will be reordered according to the computed question-passage relevance scores. If False, the original user-provided order of contexts will be preserved.</li>
|
| 496 |
+
<li><code>top_k</code> <em>(int, default: 5)</em>: if <code>reorder=True</code>, specifies the number of top-ranked passages to keep for each question.</li>
|
| 497 |
+
<li><code>enable_warnings</code> <em>(bool, default: True)</em>: whether the user prefers the warning about model usage to be printed, e.g. too long contexts or questions.</li>
|
| 498 |
+
</ul>
|
| 499 |
+
<h2 class="relative group flex items-center">
|
| 500 |
+
<a rel="nofollow" href="#model-features" class="block pr-1.5 text-lg md:absolute md:p-1.5 md:opacity-0 md:group-hover:opacity-100 md:right-full" id="model-features">
|
| 501 |
+
<span class="header-link"><svg viewBox="0 0 256 256" preserveAspectRatio="xMidYMid meet" height="1em" width="1em" role="img" aria-hidden="true" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" class="text-gray-500 hover:text-black dark:hover:text-gray-200 w-4"><path fill="currentColor" d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z"></path></svg></span>
|
| 502 |
+
</a>
|
| 503 |
+
<span>
|
| 504 |
+
Model features
|
| 505 |
+
</span>
|
| 506 |
+
</h2>
|
| 507 |
+
<ul>
|
| 508 |
+
<li><strong>XProvence natively supports 16 languages</strong>.</li>
|
| 509 |
+
<li><strong>XProvence supports 100+ languages via cross lingual transfer</strong>.</li>
|
| 510 |
+
<li><strong>XProvence encodes all sentences in the passage together</strong>: this enables capturing of coreferences between sentences and provides more accurate context pruning.</li>
|
| 511 |
+
<li><strong>XProvence automatically detects the number of sentences to keep</strong>, based on a threshold. We found that the default value of a threshold works well across various domains, but the threshold can be adjusted further to better meet the particular use case needs.</li>
|
| 512 |
+
<li><strong>XProvence works out-of-the-box with any LLM</strong>.</li>
|
| 513 |
+
</ul>
|
| 514 |
+
<h2 class="relative group flex items-center">
|
| 515 |
+
<a rel="nofollow" href="#model-details" class="block pr-1.5 text-lg md:absolute md:p-1.5 md:opacity-0 md:group-hover:opacity-100 md:right-full" id="model-details">
|
| 516 |
+
<span class="header-link"><svg viewBox="0 0 256 256" preserveAspectRatio="xMidYMid meet" height="1em" width="1em" role="img" aria-hidden="true" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" class="text-gray-500 hover:text-black dark:hover:text-gray-200 w-4"><path fill="currentColor" d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z"></path></svg></span>
|
| 517 |
+
</a>
|
| 518 |
+
<span>
|
| 519 |
+
Model Details
|
| 520 |
+
</span>
|
| 521 |
+
</h2>
|
| 522 |
+
<ul>
|
| 523 |
+
<li>Input: user question (e.g., a sentence) + retrieved context passage (e.g., a paragraph)</li>
|
| 524 |
+
<li>Output: pruned context passage, i.e., irrelevant sentences are removed + relevance score (can be used for reranking)</li>
|
| 525 |
+
<li>Model Architecture: The model was initialized from <a rel="nofollow" href="https://huggingface.co/BAAI/bge-reranker-v2-m3">bge-reranker-v2-m3</a> and finetuned with two objectives: (1) output a binary mask which can be used to prune irrelevant sentences; and (2) preserve initial reranking capabilities. </li>
|
| 526 |
+
<li>Training data: Training queries from <a rel="nofollow" href="https://huggingface.co/datasets/miracl/miracl">MIRACL</a> and documents from <a rel="nofollow" href="https://huggingface.co/datasets/wikimedia/wikipedia">Wikipedia</a>, with synthetic silver labelling of which sentences to keep, produced using <a rel="nofollow" href="https://huggingface.co/CohereLabs/aya-expanse-8b">aya-expanse-8b</a>.</li>
|
| 527 |
+
<li>Languages covered: Arabic, Bengali, English, Spanish, Persian, Finnish, France, Hindi, Indonesian, Japanese, Korean, Russian, Swahili, Telugu, Thai, Chinese</li>
|
| 528 |
+
<li>Context length: 8192 tokens (similar to the pretrained BGE-m3 model)</li>
|
| 529 |
+
<li>Evaluation: we evaluate XProvence on 26 languages from 6 different datasets. We find that XProvence is able to prune irrelevant sentences with little-to-no drop in performance, on all languages, and outperforms existing baselines on the Pareto front.</li>
|
| 530 |
+
</ul>
|
| 531 |
+
<h2 class="relative group flex items-center">
|
| 532 |
+
<a rel="nofollow" href="#license" class="block pr-1.5 text-lg md:absolute md:p-1.5 md:opacity-0 md:group-hover:opacity-100 md:right-full" id="license">
|
| 533 |
+
<span class="header-link"><svg viewBox="0 0 256 256" preserveAspectRatio="xMidYMid meet" height="1em" width="1em" role="img" aria-hidden="true" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" class="text-gray-500 hover:text-black dark:hover:text-gray-200 w-4"><path fill="currentColor" d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z"></path></svg></span>
|
| 534 |
+
</a>
|
| 535 |
+
<span>
|
| 536 |
+
License
|
| 537 |
+
</span>
|
| 538 |
+
</h2>
|
| 539 |
+
<p>This work is licensed under CC BY-NC-ND 4.0. </p>
|
| 540 |
+
<!-- HTML_TAG_END --></div>
|
| 541 |
+
|
| 542 |
+
</div></section></div></main>
|
| 543 |
+
|
| 544 |
+
</div>
|
| 545 |
+
|
| 546 |
+
<script>
|
| 547 |
+
import("\/front\/build\/kube-ad7df0d\/index.js");
|
| 548 |
+
window.moonSha = "kube-ad7df0d\/";
|
| 549 |
+
window.__hf_deferred = {};
|
| 550 |
+
</script>
|
| 551 |
+
|
| 552 |
+
<!-- Stripe -->
|
| 553 |
+
<script>
|
| 554 |
+
if (["hf.co", "huggingface.co"].includes(window.location.hostname)) {
|
| 555 |
+
const script = document.createElement("script");
|
| 556 |
+
script.src = "https://js.stripe.com/v3/";
|
| 557 |
+
script.async = true;
|
| 558 |
+
document.head.appendChild(script);
|
| 559 |
+
}
|
| 560 |
+
</script>
|
| 561 |
+
</body>
|
| 562 |
+
</html>
|
config.json
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"XProvence"
|
| 4 |
+
],
|
| 5 |
+
"attention_probs_dropout_prob": 0.1,
|
| 6 |
+
"bos_token_id": 0,
|
| 7 |
+
"classifier_dropout": null,
|
| 8 |
+
"eos_token_id": 2,
|
| 9 |
+
"hidden_act": "gelu",
|
| 10 |
+
"hidden_dropout_prob": 0.1,
|
| 11 |
+
"hidden_size": 1024,
|
| 12 |
+
"id2label": {
|
| 13 |
+
"0": "LABEL_0"
|
| 14 |
+
},
|
| 15 |
+
"initializer_range": 0.02,
|
| 16 |
+
"intermediate_size": 4096,
|
| 17 |
+
"label2id": {
|
| 18 |
+
"LABEL_0": 0
|
| 19 |
+
},
|
| 20 |
+
"layer_norm_eps": 1e-05,
|
| 21 |
+
"max_position_embeddings": 8194,
|
| 22 |
+
"model_type": "xlm-roberta",
|
| 23 |
+
"num_attention_heads": 16,
|
| 24 |
+
"num_hidden_layers": 24,
|
| 25 |
+
"output_past": true,
|
| 26 |
+
"pad_token_id": 1,
|
| 27 |
+
"position_embedding_type": "absolute",
|
| 28 |
+
"torch_dtype": "float32",
|
| 29 |
+
"transformers_version": "4.51.3",
|
| 30 |
+
"type_vocab_size": 1,
|
| 31 |
+
"use_cache": true,
|
| 32 |
+
"vocab_size": 250002,
|
| 33 |
+
"auto_map": {
|
| 34 |
+
"AutoModel": "modeling_xprovence_hf.XProvence"
|
| 35 |
+
}
|
| 36 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a02bf06d492501d02902574e8af814b4c6a16c358c9f60d7fd18e86e638c253
|
| 3 |
+
size 2271080220
|
modeling_xprovence_hf.py
ADDED
|
@@ -0,0 +1,499 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from torch import nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
from torch.nn.utils.rnn import pad_sequence
|
| 5 |
+
from torch.nn import CrossEntropyLoss, MSELoss
|
| 6 |
+
from torch.utils.data import Dataset
|
| 7 |
+
from transformers import XLMRobertaPreTrainedModel, XLMRobertaModel, PretrainedConfig, AutoTokenizer
|
| 8 |
+
from transformers.modeling_outputs import ModelOutput
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
from typing import Optional, Union, Tuple, List
|
| 11 |
+
import warnings
|
| 12 |
+
import numpy as np
|
| 13 |
+
from tqdm import tqdm
|
| 14 |
+
import string
|
| 15 |
+
|
| 16 |
+
import spacy
|
| 17 |
+
nlp = spacy.load("xx_sent_ud_sm")
|
| 18 |
+
|
| 19 |
+
@dataclass
|
| 20 |
+
class RankingCompressionOutput(ModelOutput):
|
| 21 |
+
|
| 22 |
+
loss: Optional[torch.FloatTensor] = None
|
| 23 |
+
compression_loss: Optional[torch.FloatTensor] = None
|
| 24 |
+
ranking_loss: Optional[torch.FloatTensor] = None
|
| 25 |
+
compression_logits: torch.FloatTensor = None
|
| 26 |
+
ranking_scores: torch.FloatTensor = None
|
| 27 |
+
hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 28 |
+
attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class XProvenceConfig(PretrainedConfig):
|
| 32 |
+
model_type = "XProvence"
|
| 33 |
+
def __init__(self, **kwargs):
|
| 34 |
+
super().__init__(**kwargs)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class XProvence(XLMRobertaPreTrainedModel):
|
| 38 |
+
config_class = XProvenceConfig
|
| 39 |
+
def __init__(self, config):
|
| 40 |
+
super().__init__(config)
|
| 41 |
+
num_labels = getattr(config, "num_labels", 2)
|
| 42 |
+
self.num_labels = num_labels
|
| 43 |
+
self.roberta = XLMRobertaModel(config)
|
| 44 |
+
output_dim = config.hidden_size
|
| 45 |
+
|
| 46 |
+
### RANKING LAYER
|
| 47 |
+
self.classifier = nn.Linear(output_dim, num_labels)
|
| 48 |
+
drop_out = getattr(config, "cls_dropout", None)
|
| 49 |
+
drop_out = self.config.hidden_dropout_prob if drop_out is None else drop_out
|
| 50 |
+
self.dropout = nn.Dropout(drop_out)
|
| 51 |
+
|
| 52 |
+
### COMPRESSION LAYER: another head (initialized randomly)
|
| 53 |
+
token_dropout = drop_out
|
| 54 |
+
self.token_dropout = nn.Dropout(token_dropout)
|
| 55 |
+
self.token_classifier = nn.Linear(
|
| 56 |
+
config.hidden_size, 2
|
| 57 |
+
) # => hard coded number of labels
|
| 58 |
+
|
| 59 |
+
self.tokenizer = AutoTokenizer.from_pretrained(config._name_or_path)
|
| 60 |
+
self.max_len = config.max_position_embeddings - 4
|
| 61 |
+
# Initialize weights and apply final processing
|
| 62 |
+
self.post_init()
|
| 63 |
+
|
| 64 |
+
def forward(
|
| 65 |
+
self,
|
| 66 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 67 |
+
attention_mask: Optional[torch.FloatTensor] = None,
|
| 68 |
+
token_type_ids: Optional[torch.LongTensor] = None,
|
| 69 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 70 |
+
head_mask: Optional[torch.FloatTensor] = None,
|
| 71 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 72 |
+
labels: Optional[torch.LongTensor] = None,
|
| 73 |
+
ranking_labels: Optional[torch.LongTensor] = None,
|
| 74 |
+
loss_weight: Optional[float] = None,
|
| 75 |
+
output_attentions: Optional[bool] = None,
|
| 76 |
+
output_hidden_states: Optional[bool] = None,
|
| 77 |
+
return_dict: Optional[bool] = None,
|
| 78 |
+
) -> Union[Tuple[torch.Tensor], RankingCompressionOutput]:
|
| 79 |
+
"""simplified forward"""
|
| 80 |
+
outputs = self.roberta(
|
| 81 |
+
input_ids,
|
| 82 |
+
token_type_ids=token_type_ids,
|
| 83 |
+
attention_mask=attention_mask,
|
| 84 |
+
position_ids=position_ids,
|
| 85 |
+
inputs_embeds=inputs_embeds,
|
| 86 |
+
output_attentions=output_attentions,
|
| 87 |
+
output_hidden_states=output_hidden_states,
|
| 88 |
+
return_dict=return_dict,
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
encoder_layer = outputs[0]
|
| 92 |
+
# pooled_output = self.pooler(encoder_layer)
|
| 93 |
+
pooled_output = outputs['pooler_output']
|
| 94 |
+
pooled_output = self.dropout(pooled_output)
|
| 95 |
+
ranking_logits = self.classifier(pooled_output)
|
| 96 |
+
compression_logits = self.token_classifier(self.token_dropout(encoder_layer))
|
| 97 |
+
ranking_scores = ranking_logits[:, 0].squeeze() # select first dim of logits for ranking scores
|
| 98 |
+
|
| 99 |
+
compression_loss = None
|
| 100 |
+
ranking_loss = None
|
| 101 |
+
if labels is not None:
|
| 102 |
+
# move labels to correct device to enable model parallelism
|
| 103 |
+
labels = labels.to(compression_logits.device)
|
| 104 |
+
loss_fct = CrossEntropyLoss()
|
| 105 |
+
compression_loss = loss_fct(compression_logits.view(-1, 2), labels.view(-1))
|
| 106 |
+
if ranking_labels is not None:
|
| 107 |
+
# here ranking labels are scores (from a teacher) we aim to directly distil (pointwise MSE)
|
| 108 |
+
ranking_labels = ranking_labels.to(ranking_logits.device)
|
| 109 |
+
loss_fct = MSELoss()
|
| 110 |
+
ranking_loss = loss_fct(ranking_scores, ranking_labels.squeeze())
|
| 111 |
+
loss = None
|
| 112 |
+
if (labels is not None) and (ranking_labels is not None):
|
| 113 |
+
w = loss_weight if loss_weight else 1
|
| 114 |
+
loss = compression_loss + w * ranking_loss
|
| 115 |
+
elif labels is not None:
|
| 116 |
+
loss = compression_loss
|
| 117 |
+
elif ranking_labels is not None:
|
| 118 |
+
loss = ranking_loss
|
| 119 |
+
|
| 120 |
+
return RankingCompressionOutput(
|
| 121 |
+
loss=loss,
|
| 122 |
+
compression_loss=compression_loss,
|
| 123 |
+
ranking_loss=ranking_loss,
|
| 124 |
+
compression_logits=compression_logits,
|
| 125 |
+
ranking_scores=ranking_scores,
|
| 126 |
+
hidden_states=outputs.hidden_states,
|
| 127 |
+
attentions=outputs.attentions,
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
def process(
|
| 131 |
+
self,
|
| 132 |
+
question: Union[List[str], str],
|
| 133 |
+
context: Union[List[List[str]], str],
|
| 134 |
+
title: Optional[Union[List[List[str]], str]] = "first_sentence",
|
| 135 |
+
batch_size=32,
|
| 136 |
+
threshold=0.1,
|
| 137 |
+
always_select_title=False,
|
| 138 |
+
reorder=False,
|
| 139 |
+
top_k=5,
|
| 140 |
+
enable_warnings=True,
|
| 141 |
+
):
|
| 142 |
+
|
| 143 |
+
# convert input format into queries of type List[str] and contexts/titles of type List[List[str]]
|
| 144 |
+
if type(question) == str:
|
| 145 |
+
queries = [question]
|
| 146 |
+
else: # list of strs
|
| 147 |
+
queries = question
|
| 148 |
+
if type(context) == str:
|
| 149 |
+
contexts = [[context]]
|
| 150 |
+
else:
|
| 151 |
+
contexts = context
|
| 152 |
+
if type(title) == str and title != "first_sentence":
|
| 153 |
+
titles = [[title]]
|
| 154 |
+
else:
|
| 155 |
+
titles = title
|
| 156 |
+
assert (
|
| 157 |
+
titles == "first_sentence"
|
| 158 |
+
or titles == None
|
| 159 |
+
or type(titles) == list
|
| 160 |
+
and len(titles) == len(queries)
|
| 161 |
+
), "Variable 'titles' must be 'first_sentence' or a list of strings of the same length as 'queries'"
|
| 162 |
+
if type(titles) == list:
|
| 163 |
+
assert all(
|
| 164 |
+
[
|
| 165 |
+
len(titles_item) == len(contexts_item)
|
| 166 |
+
for titles_item, contexts_item in zip(contexts, titles)
|
| 167 |
+
]
|
| 168 |
+
), "Each list in 'titles' must have the same length as the corresponding list in 'context'"
|
| 169 |
+
assert len(queries) == len(
|
| 170 |
+
contexts
|
| 171 |
+
), "Lists 'queries' and 'contexts' must have same lengths"
|
| 172 |
+
dataset = TestDataset(
|
| 173 |
+
queries=queries,
|
| 174 |
+
contexts=contexts,
|
| 175 |
+
titles=titles,
|
| 176 |
+
tokenizer=self.tokenizer,
|
| 177 |
+
max_len=self.max_len,
|
| 178 |
+
enable_warnings=enable_warnings,
|
| 179 |
+
)
|
| 180 |
+
selected_contexts = [
|
| 181 |
+
[{0: contexts[i][j]} for j in range(len(contexts[i]))]
|
| 182 |
+
for i in range(len(queries))
|
| 183 |
+
]
|
| 184 |
+
reranking_scores = [
|
| 185 |
+
[None for j in range(len(contexts[i]))] for i in range(len(queries))
|
| 186 |
+
]
|
| 187 |
+
compressions = [
|
| 188 |
+
[0 for j in range(len(contexts[i]))] for i in range(len(queries))
|
| 189 |
+
]
|
| 190 |
+
with torch.no_grad():
|
| 191 |
+
for batch_start in tqdm(
|
| 192 |
+
range(0, len(dataset), batch_size), desc="Pruning contexts..."
|
| 193 |
+
):
|
| 194 |
+
qis = dataset.qis[batch_start : batch_start + batch_size]
|
| 195 |
+
cis = dataset.cis[batch_start : batch_start + batch_size]
|
| 196 |
+
sis = dataset.sis[batch_start : batch_start + batch_size]
|
| 197 |
+
sent_coords = dataset.sent_coords[
|
| 198 |
+
batch_start : batch_start + batch_size
|
| 199 |
+
]
|
| 200 |
+
ids_list = dataset.ids[batch_start : batch_start + batch_size]
|
| 201 |
+
ids = pad_sequence(
|
| 202 |
+
ids_list, batch_first=True, padding_value=dataset.pad_idx
|
| 203 |
+
).to(self.device)
|
| 204 |
+
mask = (ids != dataset.pad_idx).to(self.device)
|
| 205 |
+
outputs = self.forward(ids, mask)
|
| 206 |
+
scores = F.softmax(outputs["compression_logits"].cpu(), dim=-1)[:, :, 1]
|
| 207 |
+
token_preds = scores > threshold
|
| 208 |
+
reranking_scrs = (
|
| 209 |
+
outputs["ranking_scores"].cpu().numpy()
|
| 210 |
+
) # get first score
|
| 211 |
+
if len(reranking_scrs.shape) == 0:
|
| 212 |
+
reranking_scrs = reranking_scrs[None]
|
| 213 |
+
for (
|
| 214 |
+
ids_list_,
|
| 215 |
+
token_preds_,
|
| 216 |
+
rerank_score,
|
| 217 |
+
qi,
|
| 218 |
+
ci,
|
| 219 |
+
si,
|
| 220 |
+
sent_coords_,
|
| 221 |
+
) in zip(
|
| 222 |
+
ids_list, token_preds, reranking_scrs, qis, cis, sis, sent_coords
|
| 223 |
+
):
|
| 224 |
+
|
| 225 |
+
selected_mask = sentence_rounding(
|
| 226 |
+
token_preds_.cpu().numpy(),
|
| 227 |
+
np.array(sent_coords_),
|
| 228 |
+
threshold=threshold,
|
| 229 |
+
always_select_title=always_select_title
|
| 230 |
+
and si == 0
|
| 231 |
+
and titles != None,
|
| 232 |
+
)
|
| 233 |
+
assert len(selected_mask) == len(token_preds_)
|
| 234 |
+
selected_contexts[qi][ci][si] = ids_list_[
|
| 235 |
+
selected_mask[: len(ids_list_)]
|
| 236 |
+
]
|
| 237 |
+
if si == 0:
|
| 238 |
+
reranking_scores[qi][ci] = rerank_score
|
| 239 |
+
for i in range(len(queries)):
|
| 240 |
+
for j in range(len(contexts[i])):
|
| 241 |
+
if type(selected_contexts[i][j][0]) != str:
|
| 242 |
+
toks = torch.cat(
|
| 243 |
+
[
|
| 244 |
+
ids_
|
| 245 |
+
for _, ids_ in sorted(
|
| 246 |
+
selected_contexts[i][j].items(), key=lambda x: x[0]
|
| 247 |
+
)
|
| 248 |
+
]
|
| 249 |
+
)
|
| 250 |
+
selected_contexts[i][j] = self.tokenizer.decode(
|
| 251 |
+
toks,
|
| 252 |
+
skip_special_tokens=True,
|
| 253 |
+
clean_up_tokenization_spaces=False,
|
| 254 |
+
)
|
| 255 |
+
else:
|
| 256 |
+
selected_contexts[i][j] = selected_contexts[i][j][0]
|
| 257 |
+
len_original = len(contexts[i][j])
|
| 258 |
+
len_compressed = len(selected_contexts[i][j])
|
| 259 |
+
compressions[i][j] = (len_original-len_compressed)/len_original * 100
|
| 260 |
+
if reorder:
|
| 261 |
+
idxs = np.argsort(reranking_scores[i])[::-1][:top_k]
|
| 262 |
+
selected_contexts[i] = [selected_contexts[i][j] for j in idxs]
|
| 263 |
+
reranking_scores[i] = [reranking_scores[i][j] for j in idxs]
|
| 264 |
+
compressions[i] = [compressions[i][j] for j in idxs]
|
| 265 |
+
|
| 266 |
+
if type(context) == str:
|
| 267 |
+
selected_contexts = selected_contexts[0][0]
|
| 268 |
+
reranking_scores = reranking_scores[0][0]
|
| 269 |
+
compressions = compressions[0][0]
|
| 270 |
+
|
| 271 |
+
return {
|
| 272 |
+
"pruned_context": selected_contexts,
|
| 273 |
+
"reranking_score": reranking_scores,
|
| 274 |
+
"compression_rate": compressions,
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
# Some utils functions
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def sentence_rounding(predictions, chunks, threshold, always_select_title=True):
|
| 282 |
+
"""
|
| 283 |
+
predictions: a binary vector containing 1 for tokens which were selected and 0s otherwise
|
| 284 |
+
chunks: a list of pairs [start, end] of sentence, i.e. sentence is in coordinates predictions[start:end]
|
| 285 |
+
the functions
|
| 286 |
+
"""
|
| 287 |
+
cumulative_sum = np.cumsum(predictions)
|
| 288 |
+
chunk_sums = cumulative_sum[chunks[:, 1] - 1] - np.where(
|
| 289 |
+
chunks[:, 0] > 0, cumulative_sum[chunks[:, 0] - 1], 0
|
| 290 |
+
)
|
| 291 |
+
chunk_lengths = chunks[:, 1] - chunks[:, 0]
|
| 292 |
+
chunk_means = chunk_sums / chunk_lengths
|
| 293 |
+
if always_select_title and (chunk_means>threshold).any():
|
| 294 |
+
chunk_means[0] = 1
|
| 295 |
+
means = np.hstack((np.zeros(1), chunk_means, np.zeros(1)))
|
| 296 |
+
repeats = np.hstack(
|
| 297 |
+
([chunks[0][0]], chunk_lengths, [predictions.shape[0] - chunks[-1][1]])
|
| 298 |
+
)
|
| 299 |
+
return np.repeat(means, repeats) > threshold
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def normalize(s: str) -> str:
|
| 303 |
+
def white_space_fix(text):
|
| 304 |
+
return " ".join(text.split())
|
| 305 |
+
|
| 306 |
+
def remove_punc(text):
|
| 307 |
+
exclude = set(string.punctuation)
|
| 308 |
+
return "".join(ch for ch in text if ch not in exclude)
|
| 309 |
+
|
| 310 |
+
def lower(text):
|
| 311 |
+
return text.lower()
|
| 312 |
+
|
| 313 |
+
return white_space_fix(remove_punc(lower(s)))
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
def sent_split_and_tokenize(text, tokenizer, max_len):
|
| 317 |
+
# sents_nltk = nltk.sent_tokenize(text)
|
| 318 |
+
sents_nltk = [sent.text.strip() for sent in nlp(text).sents]
|
| 319 |
+
sents = []
|
| 320 |
+
for j, sent_nltk in enumerate(sents_nltk):
|
| 321 |
+
tokinput = (" " if j != 0 else "") + sent_nltk
|
| 322 |
+
tok = tokenizer.encode(tokinput, add_special_tokens=False)
|
| 323 |
+
ltok = len(tok)
|
| 324 |
+
if ltok == 0:
|
| 325 |
+
continue
|
| 326 |
+
if ltok <= max_len:
|
| 327 |
+
sents.append(tok)
|
| 328 |
+
else:
|
| 329 |
+
for begin in range(0, ltok, max_len):
|
| 330 |
+
sents.append(tok[begin:begin+max_len])
|
| 331 |
+
return sents
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
class TestDataset(Dataset):
|
| 335 |
+
def __init__(
|
| 336 |
+
self,
|
| 337 |
+
queries,
|
| 338 |
+
contexts,
|
| 339 |
+
tokenizer,
|
| 340 |
+
max_len=6000,
|
| 341 |
+
titles="first_sentence",
|
| 342 |
+
enable_warnings=True,
|
| 343 |
+
):
|
| 344 |
+
self.tokenizer = tokenizer
|
| 345 |
+
self.max_len = max_len
|
| 346 |
+
self.pad_idx = self.tokenizer.pad_token_id
|
| 347 |
+
self.cls_idx = [self.tokenizer.cls_token_id]
|
| 348 |
+
self.sep_idx = [self.tokenizer.sep_token_id]
|
| 349 |
+
self.eos = [self.tokenizer.eos_token_id]
|
| 350 |
+
# hardcoded deberta-specific indexes
|
| 351 |
+
self.nb_spe_tok = len(self.cls_idx) + len(self.sep_idx)
|
| 352 |
+
self.enable_warnings = enable_warnings
|
| 353 |
+
self.unusual_query_length = (
|
| 354 |
+
self.max_len // 2
|
| 355 |
+
) # TODO: change to data-driven value
|
| 356 |
+
self.unusual_title_len = self.max_len // 2 # TODO: change to data-driven value
|
| 357 |
+
self.create_dataset(queries, contexts, titles)
|
| 358 |
+
self.len = len(self.cis)
|
| 359 |
+
|
| 360 |
+
def create_dataset(self, queries, contexts, titles="first_sentence"):
|
| 361 |
+
self.qis = []
|
| 362 |
+
self.cis = []
|
| 363 |
+
self.sis = []
|
| 364 |
+
self.sent_coords = []
|
| 365 |
+
self.cntx_coords = []
|
| 366 |
+
self.ids = []
|
| 367 |
+
if self.enable_warnings:
|
| 368 |
+
warnings_dict = {
|
| 369 |
+
"zero_len_query": set(),
|
| 370 |
+
"too_long_query": set(),
|
| 371 |
+
"unusually_long_query": set(),
|
| 372 |
+
"unusually_long_title": set(),
|
| 373 |
+
"split_context": set(),
|
| 374 |
+
}
|
| 375 |
+
for i, query in enumerate(queries):
|
| 376 |
+
tokenized_query = self.tokenizer.encode(
|
| 377 |
+
normalize(query), add_special_tokens=False
|
| 378 |
+
)
|
| 379 |
+
# normalize query because all training data has normalized queries
|
| 380 |
+
query_len = len(tokenized_query)
|
| 381 |
+
if query_len == 0:
|
| 382 |
+
if self.enable_warnings:
|
| 383 |
+
warnings_dict["zero_len_query"].add(i)
|
| 384 |
+
continue
|
| 385 |
+
elif query_len >= self.max_len - self.nb_spe_tok - 1: # -1 for eos
|
| 386 |
+
if self.enable_warnings:
|
| 387 |
+
warnings_dict["too_long_query"].add(i)
|
| 388 |
+
continue
|
| 389 |
+
elif query_len >= self.unusual_query_length:
|
| 390 |
+
if self.enable_warnings:
|
| 391 |
+
warnings_dict["unusually_long_query"].add(i)
|
| 392 |
+
left_0 = len(tokenized_query) + self.nb_spe_tok
|
| 393 |
+
tokenized_seq_0 = self.cls_idx + tokenized_query + self.sep_idx
|
| 394 |
+
max_len = self.max_len - left_0 - 1
|
| 395 |
+
for j, cntx in enumerate(contexts[i]):
|
| 396 |
+
title = titles[i][j] if type(titles) == list else titles
|
| 397 |
+
tokenized_sents = sent_split_and_tokenize(cntx, self.tokenizer, max_len)
|
| 398 |
+
# each (sent + query + special tokens) <= max_len
|
| 399 |
+
if title is not None and title != "first_sentence":
|
| 400 |
+
tokenized_title = self.tokenizer.encode(
|
| 401 |
+
title, add_special_tokens=False
|
| 402 |
+
)
|
| 403 |
+
ltok = len(tokenized_title)
|
| 404 |
+
if ltok == 0:
|
| 405 |
+
pass
|
| 406 |
+
elif ltok <= max_len:
|
| 407 |
+
tokenized_sents = [tokenized_title] + tokenized_sents
|
| 408 |
+
else:
|
| 409 |
+
if self.enable_warnings and ltok >= self.unusual_title_len:
|
| 410 |
+
warnings_dict["unusually_long_title"].add(i)
|
| 411 |
+
tokenized_sents = [
|
| 412 |
+
tokenized_title[begin : begin + max_len]
|
| 413 |
+
for begin in range(0, ltok, max_len)
|
| 414 |
+
] + tokenized_sents
|
| 415 |
+
tokenized_seq = tokenized_seq_0
|
| 416 |
+
left = left_0
|
| 417 |
+
sent_coords = []
|
| 418 |
+
block = 0
|
| 419 |
+
for idx, tokenized_sent in enumerate(tokenized_sents):
|
| 420 |
+
l = len(tokenized_sent)
|
| 421 |
+
if left + l <= self.max_len - 1:
|
| 422 |
+
sent_coords.append([left, left + l])
|
| 423 |
+
tokenized_seq = tokenized_seq + tokenized_sent
|
| 424 |
+
left += l
|
| 425 |
+
else:
|
| 426 |
+
if self.enable_warnings:
|
| 427 |
+
warnings_dict["split_context"].add(i)
|
| 428 |
+
if len(tokenized_seq) > left_0:
|
| 429 |
+
tokenized_seq = tokenized_seq + self.eos
|
| 430 |
+
self.qis.append(i)
|
| 431 |
+
self.cis.append(j)
|
| 432 |
+
self.sis.append(block)
|
| 433 |
+
self.sent_coords.append(sent_coords)
|
| 434 |
+
self.cntx_coords.append(
|
| 435 |
+
[sent_coords[0][0], sent_coords[-1][1]]
|
| 436 |
+
)
|
| 437 |
+
self.ids.append(torch.tensor(tokenized_seq))
|
| 438 |
+
tokenized_seq = tokenized_seq_0 + tokenized_sent
|
| 439 |
+
sent_coords = [[left_0, left_0 + l]]
|
| 440 |
+
left = left_0 + l
|
| 441 |
+
block += 1
|
| 442 |
+
if len(tokenized_seq) > left_0:
|
| 443 |
+
tokenized_seq = tokenized_seq + self.eos
|
| 444 |
+
self.qis.append(i)
|
| 445 |
+
self.cis.append(j)
|
| 446 |
+
self.sis.append(block)
|
| 447 |
+
self.sent_coords.append(sent_coords)
|
| 448 |
+
self.cntx_coords.append([sent_coords[0][0], sent_coords[-1][1]])
|
| 449 |
+
self.ids.append(torch.tensor(tokenized_seq))
|
| 450 |
+
if self.enable_warnings:
|
| 451 |
+
self.print_warnings(warnings_dict, len(queries))
|
| 452 |
+
|
| 453 |
+
def __len__(self):
|
| 454 |
+
return len(self.ids)
|
| 455 |
+
|
| 456 |
+
def print_warnings(self, warnings_dict, N):
|
| 457 |
+
n = len(warnings_dict["zero_len_query"])
|
| 458 |
+
info = " You can suppress Provence warnings by setting enable_warnings=False."
|
| 459 |
+
if n > 0:
|
| 460 |
+
ex = list(warnings_dict["zero_len_query"])[:10]
|
| 461 |
+
warnings.warn(
|
| 462 |
+
f"{n} out of {N} queries have zero length, e.g. at indexes {ex}. "
|
| 463 |
+
"These examples will be skipped in context pruning, "
|
| 464 |
+
"their contexts will be kept as is." + info
|
| 465 |
+
)
|
| 466 |
+
n = len(warnings_dict["too_long_query"])
|
| 467 |
+
if n > 0:
|
| 468 |
+
ex = list(warnings_dict["too_long_query"])[:10]
|
| 469 |
+
warnings.warn(
|
| 470 |
+
f"{n} out of {N} queries are too long for context length {self.max_len}, "
|
| 471 |
+
f"e.g. at indexes {ex}. These examples will be skipped in context pruning, "
|
| 472 |
+
"their contexts will be kept as is." + info
|
| 473 |
+
)
|
| 474 |
+
n = len(warnings_dict["unusually_long_query"])
|
| 475 |
+
if n > 0:
|
| 476 |
+
ex = list(warnings_dict["unusually_long_query"])[:10]
|
| 477 |
+
warnings.warn(
|
| 478 |
+
f"{n} out of {N} queries are longer than {self.unusual_query_length} tokens, "
|
| 479 |
+
f"e.g. at indexes {ex}. These examples will processed as usual in context pruning, "
|
| 480 |
+
"but the quality of context pruning could be reduced." + info
|
| 481 |
+
)
|
| 482 |
+
n = len(warnings_dict["unusually_long_title"])
|
| 483 |
+
if n > 0:
|
| 484 |
+
ex = list(warnings_dict["unusually_long_title"])[:10]
|
| 485 |
+
warnings.warn(
|
| 486 |
+
f"{n} out of {N} titles are longer than {self.unusual_title_length} tokens, "
|
| 487 |
+
f"e.g. at indexes {ex}. These examples will processed as usual in context pruning, "
|
| 488 |
+
"but the quality of context pruning could be reduced." + info
|
| 489 |
+
)
|
| 490 |
+
n = len(warnings_dict["split_context"])
|
| 491 |
+
if n > 0:
|
| 492 |
+
ex = list(warnings_dict["split_context"])[:10]
|
| 493 |
+
warnings.warn(
|
| 494 |
+
f"{n} out of {N} contexts were split into several pieces for context pruning, "
|
| 495 |
+
f"due to a limited context length of Provence which is equal to {self.max_len}. "
|
| 496 |
+
"This could potentially reduce the quality of context pruning. "
|
| 497 |
+
"You could consider checking and reducing lengths of contexts, queries, or titles."
|
| 498 |
+
+ info
|
| 499 |
+
)
|
sentencepiece.bpe.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
|
| 3 |
+
size 5069051
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<s>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"cls_token": {
|
| 10 |
+
"content": "<s>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"eos_token": {
|
| 17 |
+
"content": "</s>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"mask_token": {
|
| 24 |
+
"content": "<mask>",
|
| 25 |
+
"lstrip": true,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
},
|
| 30 |
+
"pad_token": {
|
| 31 |
+
"content": "<pad>",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": false,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false
|
| 36 |
+
},
|
| 37 |
+
"sep_token": {
|
| 38 |
+
"content": "</s>",
|
| 39 |
+
"lstrip": false,
|
| 40 |
+
"normalized": false,
|
| 41 |
+
"rstrip": false,
|
| 42 |
+
"single_word": false
|
| 43 |
+
},
|
| 44 |
+
"unk_token": {
|
| 45 |
+
"content": "<unk>",
|
| 46 |
+
"lstrip": false,
|
| 47 |
+
"normalized": false,
|
| 48 |
+
"rstrip": false,
|
| 49 |
+
"single_word": false
|
| 50 |
+
}
|
| 51 |
+
}
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5d704acb9b49c8315d60393b34ec989aa0d6c2e3e0682f6b5cf6ab79649ca685
|
| 3 |
+
size 17083151
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"added_tokens_decoder": {
|
| 3 |
+
"0": {
|
| 4 |
+
"content": "<s>",
|
| 5 |
+
"lstrip": false,
|
| 6 |
+
"normalized": false,
|
| 7 |
+
"rstrip": false,
|
| 8 |
+
"single_word": false,
|
| 9 |
+
"special": true
|
| 10 |
+
},
|
| 11 |
+
"1": {
|
| 12 |
+
"content": "<pad>",
|
| 13 |
+
"lstrip": false,
|
| 14 |
+
"normalized": false,
|
| 15 |
+
"rstrip": false,
|
| 16 |
+
"single_word": false,
|
| 17 |
+
"special": true
|
| 18 |
+
},
|
| 19 |
+
"2": {
|
| 20 |
+
"content": "</s>",
|
| 21 |
+
"lstrip": false,
|
| 22 |
+
"normalized": false,
|
| 23 |
+
"rstrip": false,
|
| 24 |
+
"single_word": false,
|
| 25 |
+
"special": true
|
| 26 |
+
},
|
| 27 |
+
"3": {
|
| 28 |
+
"content": "<unk>",
|
| 29 |
+
"lstrip": false,
|
| 30 |
+
"normalized": false,
|
| 31 |
+
"rstrip": false,
|
| 32 |
+
"single_word": false,
|
| 33 |
+
"special": true
|
| 34 |
+
},
|
| 35 |
+
"250001": {
|
| 36 |
+
"content": "<mask>",
|
| 37 |
+
"lstrip": true,
|
| 38 |
+
"normalized": false,
|
| 39 |
+
"rstrip": false,
|
| 40 |
+
"single_word": false,
|
| 41 |
+
"special": true
|
| 42 |
+
}
|
| 43 |
+
},
|
| 44 |
+
"bos_token": "<s>",
|
| 45 |
+
"clean_up_tokenization_spaces": true,
|
| 46 |
+
"cls_token": "<s>",
|
| 47 |
+
"eos_token": "</s>",
|
| 48 |
+
"extra_special_tokens": {},
|
| 49 |
+
"mask_token": "<mask>",
|
| 50 |
+
"model_max_length": 8192,
|
| 51 |
+
"pad_token": "<pad>",
|
| 52 |
+
"sep_token": "</s>",
|
| 53 |
+
"sp_model_kwargs": {},
|
| 54 |
+
"tokenizer_class": "XLMRobertaTokenizer",
|
| 55 |
+
"unk_token": "<unk>"
|
| 56 |
+
}
|