"""Romansh logic: location -> idiom lookup and translation. Uses the ZurichNLP Romansh NLLB-200 1.3B model (University of Zurich / Lia Rumantscha) to translate a set of everyday phrases into the local Romansh idiom. """ import json import math import re import threading import unicodedata from functools import lru_cache from pathlib import Path import requests MODEL_ID = "ZurichNLP/romansh-nllb-1.3b-ct2" IDIOMS = { "rm-sursilv": "Sursilvan", "rm-sutsilv": "Sutsilvan", "rm-surmiran": "Surmiran", "rm-puter": "Puter", "rm-vallader": "Vallader", "rm-rumgr": "Rumantsch Grischun", } # "Grüezi!" -> "Allegra!" in Pledari Grond (RG, Surmiran); understood in every idiom. GREETING = "Allegra!" # (names, lat, lon, idiom). The first name is used for display. None = Romansh is not the local language. PLACES = [ # Sursilvan (["Disentis/Mustér", "Disentis", "Mustér", "Muster"], 46.703, 8.853, "rm-sursilv"), (["Sedrun", "Tujetsch"], 46.680, 8.775, "rm-sursilv"), (["Sumvitg", "Somvix"], 46.728, 8.938, "rm-sursilv"), (["Trun"], 46.743, 8.988, "rm-sursilv"), (["Breil/Brigels", "Breil", "Brigels"], 46.770, 9.064, "rm-sursilv"), (["Ilanz/Glion", "Ilanz", "Glion"], 46.774, 9.205, "rm-sursilv"), (["Vella", "Lumnezia", "Lugnez"], 46.717, 9.170, "rm-sursilv"), (["Lumbrein"], 46.684, 9.137, "rm-sursilv"), (["Vrin"], 46.655, 9.100, "rm-sursilv"), (["Vignogn"], 46.698, 9.148, "rm-sursilv"), (["Waltensburg/Vuorz", "Vuorz", "Waltensburg"], 46.775, 9.110, "rm-sursilv"), (["Rueun", "Ruis"], 46.778, 9.150, "rm-sursilv"), (["Schluein"], 46.789, 9.226, "rm-sursilv"), (["Laax", "Lags"], 46.807, 9.258, "rm-sursilv"), (["Flims", "Flem"], 46.835, 9.284, "rm-sursilv"), (["Falera"], 46.801, 9.233, "rm-sursilv"), (["Trin"], 46.828, 9.363, "rm-sursilv"), (["Domat/Ems", "Domat", "Ems"], 46.835, 9.450, "rm-sursilv"), (["Obersaxen", "Sursaissa"], 46.747, 9.107, "rm-sursilv"), # Sutsilvan (["Andeer"], 46.603, 9.426, "rm-sutsilv"), (["Zillis/Ziràn", "Zillis", "Ziran"], 46.634, 9.444, "rm-sutsilv"), (["Donat"], 46.628, 9.430, "rm-sutsilv"), (["Casti-Wergenstein"], 46.629, 9.408, "rm-sutsilv"), (["Domleschg/Tumleastga", "Tomils", "Tumegl", "Domleschg"], 46.763, 9.442, "rm-sutsilv"), (["Scharans/Scharons", "Scharans"], 46.717, 9.467, "rm-sutsilv"), (["Thusis/Tusàn", "Thusis", "Tusan"], 46.697, 9.440, "rm-sutsilv"), # Surmiran (["Savognin"], 46.597, 9.598, "rm-surmiran"), (["Tiefencastel/Casti", "Tiefencastel", "Casti", "Albula/Alvra"], 46.660, 9.578, "rm-surmiran"), (["Lantsch/Lenz", "Lantsch", "Lenz"], 46.683, 9.563, "rm-surmiran"), (["Vaz/Obervaz", "Lenzerheide", "Lai", "Valbella"], 46.722, 9.556, "rm-surmiran"), (["Salouf"], 46.618, 9.576, "rm-surmiran"), (["Mulegns"], 46.523, 9.620, "rm-surmiran"), (["Bivio/Beiva", "Bivio"], 46.469, 9.651, "rm-surmiran"), (["Bergün/Bravuogn", "Bergün", "Bergun", "Bravuogn"], 46.630, 9.746, "rm-surmiran"), (["Surses", "Oberhalbstein"], 46.560, 9.610, "rm-surmiran"), # Puter (["St. Moritz/San Murezzan", "St. Moritz", "Sankt Moritz", "San Murezzan", "St Moritz"], 46.498, 9.839, "rm-puter"), (["Celerina/Schlarigna", "Celerina", "Schlarigna"], 46.512, 9.859, "rm-puter"), (["Samedan"], 46.534, 9.873, "rm-puter"), (["Pontresina"], 46.491, 9.902, "rm-puter"), (["Bever"], 46.551, 9.889, "rm-puter"), (["La Punt Chamues-ch", "La Punt"], 46.582, 9.927, "rm-puter"), (["Madulain"], 46.587, 9.939, "rm-puter"), (["Zuoz"], 46.602, 9.960, "rm-puter"), (["S-chanf"], 46.614, 9.987, "rm-puter"), (["Silvaplana"], 46.459, 9.795, "rm-puter"), (["Sils/Segl", "Sils", "Segl", "Sils Maria"], 46.430, 9.758, "rm-puter"), # Vallader (["Zernez"], 46.698, 10.093, "rm-vallader"), (["Susch"], 46.751, 10.080, "rm-vallader"), (["Lavin"], 46.770, 10.110, "rm-vallader"), (["Guarda"], 46.775, 10.151, "rm-vallader"), (["Ardez"], 46.776, 10.200, "rm-vallader"), (["Ftan"], 46.795, 10.249, "rm-vallader"), (["Scuol", "Schuls"], 46.797, 10.298, "rm-vallader"), (["Sent"], 46.817, 10.338, "rm-vallader"), (["Ramosch"], 46.834, 10.383, "rm-vallader"), (["Tschlin"], 46.869, 10.428, "rm-vallader"), (["Tschierv"], 46.624, 10.343, "rm-vallader"), (["Valchava"], 46.597, 10.406, "rm-vallader"), (["Sta. Maria Val Müstair", "Santa Maria", "Sta. Maria"], 46.602, 10.423, "rm-vallader"), (["Müstair", "Mustair", "Val Müstair"], 46.630, 10.447, "rm-vallader"), # Graubünden places where Romansh is not the main local language (["Chur/Cuira", "Chur", "Cuira", "Coira"], 46.849, 9.532, None), (["Landquart"], 46.967, 9.555, None), (["Davos/Tavau", "Davos", "Tavau"], 46.802, 9.836, None), (["Klosters"], 46.869, 9.879, None), (["Arosa"], 46.778, 9.677, None), (["Splügen"], 46.551, 9.322, None), (["Vals"], 46.616, 9.180, None), (["Samnaun"], 46.944, 10.360, None), (["Poschiavo/Puschlav", "Poschiavo", "Puschlav"], 46.325, 10.058, None), (["Vicosoprano", "Bregaglia", "Bergell"], 46.354, 9.626, None), (["Mesocco/Misox", "Mesocco", "Misox"], 46.394, 9.233, None), (["Roveredo"], 46.237, 9.127, None), ] GR_BBOX = (46.15, 47.07, 8.65, 10.50) # lat_min, lat_max, lon_min, lon_max def _norm(s: str) -> str: s = unicodedata.normalize("NFKD", s or "").encode("ascii", "ignore").decode() return " ".join(s.lower().replace("-", " ").replace(".", " ").split()) _NAME_INDEX = {_norm(n): p for p in PLACES for n in p[0]} def _haversine(lat1, lon1, lat2, lon2): r = 6371.0 p1, p2 = math.radians(lat1), math.radians(lat2) dp, dl = p2 - p1, math.radians(lon2 - lon1) a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2 return 2 * r * math.asin(math.sqrt(a)) def nearest_place(lat, lon): return min(((p, _haversine(lat, lon, p[1], p[2])) for p in PLACES), key=lambda x: x[1]) def idioms_by_distance(lat, lon, first=None): """The five idioms with `first` (the local one) at the front, the others ordered by how close their nearest village is; Rumantsch Grischun, the written standard, comes last.""" regional = [code for code in IDIOMS if code != "rm-rumgr"] if lat is not None and lon is not None: dist = {code: min((_haversine(lat, lon, p[1], p[2]) for p in PLACES if p[3] == code), default=float("inf")) for code in regional} regional.sort(key=dist.get) if first in regional: regional.remove(first) regional.insert(0, first) return [*regional, "rm-rumgr"] SEARCH_URL = "https://api3.geo.admin.ch/rest/services/api/SearchServer" # Municipalities first, then postcodes, then other named places (localities before e.g. passes or buildings) _ORIGIN_RANK = {"gg25": 0, "zipcode": 1, "gazetteer": 2} _UMLAUTS = {"ae": "ä", "oe": "ö", "ue": "ü"} def _search_name(label): """Place name from a swisstopo search label like 'Ort Ftan (GR) - Scuol' or 'Chur (GR)'.""" m = re.search(r"([^<]*)", label) name = m.group(1) if m else re.sub(r"<[^>]+>", "", label) return re.sub(r"\s*\([A-Z]{2}\)$", "", name).strip() def _swisstopo_search(text): r = requests.get(SEARCH_URL, params={"searchText": text, "type": "locations", "origins": "gg25,zipcode,gazetteer", "limit": 15, "sr": 4326}, timeout=10) r.raise_for_status() return r.json().get("results", []) @lru_cache(maxsize=512) def geocode(city: str): """Look up a Swiss place name or postcode with the swisstopo search service (geo.admin.ch). The service also returns loose matches, so only a hit whose name is the typed name (or starts with it) counts; exact names win over longer ones, then municipalities over postcodes over other places. Returns (lat, lon, name) or None. """ q = _norm(city) if not q: return None queries = [city] if any(k in q for k in _UMLAUTS): # "Zuerich" -> also try "Zürich" queries.append(re.sub("ae|oe|ue", lambda m: _UMLAUTS[m.group(0)], city.lower())) best = None try: for text in queries: for hit in _swisstopo_search(text): a = hit["attrs"] name = _search_name(a.get("label", "")) names = [_norm(n) for n in re.split(r"/| - ", name)] if a.get("origin") == "zipcode": names.append(_norm(re.sub(r"^\d{4}\s*-?\s*", "", name))) names = [n for n in names if n] variants = {q, _norm(text)} exact = any(n in variants for n in names) if not exact and not any(n.startswith(v + " ") for n in names for v in variants): continue locality = a.get("origin") != "gazetteer" or "Ort" in a.get("label", "") key = (not exact, _ORIGIN_RANK.get(a.get("origin"), 9), not locality) if best is None or key < best[0]: best = (key, float(a["lat"]), float(a["lon"]), name) if best: break except Exception: return None return best[1:] if best else None def in_graubuenden(lat, lon): return GR_BBOX[0] <= lat <= GR_BBOX[1] and GR_BBOX[2] <= lon <= GR_BBOX[3] def resolve_location(lat, lon, city): """Return (idiom_code, info). info holds language-neutral facts that i18n turns into text: src: "town" | "geo" | "device" | None where the location came from status: "local" | "non_rm" | "outside" | "notfound" | "noloc" place, near, city, label: details for the message """ city = (city or "").strip() info = {"src": None, "city": city} if city: place = _NAME_INDEX.get(_norm(city)) if place: return _idiom_for(place, 0.0, {**info, "src": "town"}) hit = geocode(city) if not hit: return "rm-rumgr", {**info, "status": "notfound"} lat, lon, label = hit info.update(src="geo", label=label.split(",")[0], lat=lat, lon=lon) elif lat is not None and lon is not None: info.update(src="device", lat=lat, lon=lon) else: return "rm-rumgr", {**info, "status": "noloc"} if not in_graubuenden(lat, lon): return "rm-rumgr", {**info, "status": "outside", "lat": lat, "lon": lon} place, dist = nearest_place(lat, lon) return _idiom_for(place, dist, info) def _idiom_for(place, dist, info): idiom = place[3] info = {**info, "place": place[0][0], "near": dist > 1.5, "lat": info.get("lat", place[1]), "lon": info.get("lon", place[2])} if idiom is None: return "rm-rumgr", {**info, "status": "non_rm"} return idiom, {**info, "status": "local"} # ---------------------------------------------------------------- phrases # Built by tools/build_phrases.py from Pledari Grond (Lia Rumantscha) dictionary entries, ranked by # how often each form is used as a greeting in ZurichNLP's Romansh corpora. Puter and Vallader # (no queryable dictionary) come from the ZurichNLP model, ranked the same way. _PHRASES = json.loads((Path(__file__).parent / "phrases.json").read_text(encoding="utf-8")) def phrases_for(idiom): """The 9 phrases after the greeting, as dicts with 'text' and 'source' ('pledarigrond' | 'model').""" return _PHRASES[idiom] # ---------------------------------------------------------------- model (free-text translation only) _model = None _lock = threading.Lock() def _load_model(): global _model if _model is None: import ctranslate2 from huggingface_hub import snapshot_download from transformers import NllbTokenizer path = snapshot_download(MODEL_ID) _model = (NllbTokenizer.from_pretrained(path), ctranslate2.Translator(path, device="cpu", compute_type="int8")) return _model def translate(sentences, src, tgt, beam_size=4): with _lock: # loading and the tokenizer's source language are shared state tokenizer, translator = _load_model() tokenizer.src_lang = src tokenizer.set_src_lang_special_tokens(src) batch = [tokenizer.convert_ids_to_tokens(tokenizer.encode(s)) for s in sentences] results = translator.translate_batch( batch, target_prefix=[[tgt]] * len(batch), beam_size=beam_size, max_decoding_length=256 ) return [ tokenizer.decode(tokenizer.convert_tokens_to_ids(r.hypotheses[0][1:]), skip_special_tokens=True) for r in results ] def idiom_code(name): return next(k for k, v in IDIOMS.items() if v == name) def phrase_rows(idiom): """The 10 phrases for an idiom as (romansh, verified, key); meanings live in i18n.MEANINGS.""" return [(GREETING, True, "hello"), *((p["text"], p["source"] == "pledarigrond", p["key"]) for p in phrases_for(idiom))]