use new file locations for translations

- translation files are now stored in a separate repo, and
use a layout compatible with Pontoon
- normalize the language code in aqt, so that old config settings
and command line arguments are correctly handled
- store Qt and gettext translations in separate subfolders
- remove Crowdin scripts

pylib/anki/lang.py

@@ -106,6 +106,33 @@ compatMap = {
     "vi": "vi_VN",
 }
 
+
+def lang_to_disk_lang(lang: str) -> str:
+    """Normalize lang, then convert it to name used on disk."""
+    # convert it into our canonical representation first
+    lang = lang.replace("-", "_")
+    if lang in compatMap:
+        lang = compatMap[lang]
+
+    # these language/region combinations are fully qualified, but with a hyphen
+    if lang in (
+        "en_GB",
+        "es_ES",
+        "ga_IE",
+        "hy_AM",
+        "nb_NO",
+        "nn_NO",
+        "pt_BR",
+        "pt_PT",
+        "sv_SE",
+        "zh_CN",
+        "zh_TW",
+    ):
+        return lang.replace("_", "-")
+    # other languages have the region portion stripped
+    return re.match("(.*)_", lang).group(1)
+
+
 threadLocal = threading.local()
 
 # global defaults
@@ -131,7 +158,6 @@ def ngettext(single: str, plural: str, n: int) -> str:
 
 
 def setLang(lang: str, locale_dir: str, local: bool = True) -> None:
-    lang = mungeCode(lang)
     trans = gettext.translation("anki", locale_dir, languages=[lang], fallback=True)
     if local:
         threadLocal.currentLang = lang
@@ -157,13 +183,5 @@ def noHint(str) -> str:
     return re.sub(r"(^.*?)( ?\(.+?\))?$", "\\1", str)
 
 
-def mungeCode(code: str) -> Any:
-    code = code.replace("-", "_")
-    if code in compatMap:
-        code = compatMap[code]
-
-    return code
-
-
 if not currentTranslation:
     setLang("en_US", locale_dir="", local=False)

qt/Makefile

@@ -25,8 +25,8 @@ all: check
 	./tools/build_ui.sh
 	@touch $@
 
-.build/i18n: $(wildcard i18n/translations/anki.pot/*)
-	(cd i18n && ./build-mo-files && ./copy-qt-files)
+.build/i18n: i18n/po $(wildcard i18n/po/desktop/*/anki.po)
+	(cd i18n && ./pull-git && ./build-mo-files && ./copy-qt-files)
 	@touch $@
 
 TSDEPS := $(wildcard ts/src/*.ts) $(wildcard ts/scss/*.scss)

qt/aqt/__init__.py

@@ -148,8 +148,8 @@ def setupLang(
         locale.setlocale(locale.LC_ALL, "")
     except:
         pass
-    lang = force or pm.meta["defaultLang"]
 
+    # add _ and ngettext globals used by legacy code
     def fn__(arg):
         print("accessing _ without importing from anki.lang will break in the future")
         print("".join(traceback.format_stack()[-2]))
@@ -168,15 +168,27 @@ def setupLang(
 
     builtins.__dict__["_"] = fn__
     builtins.__dict__["ngettext"] = fn_ngettext
+
+    # get lang and normalize into ja/zh-CN form
+    lang = force or pm.meta["defaultLang"]
+    lang = anki.lang.lang_to_disk_lang(lang)
+
+    # load gettext catalog
     ldir = locale_dir()
-    anki.lang.setLang(lang, ldir, local=False)
+    gettext_dir = os.path.join(ldir, "gettext")
+    anki.lang.setLang(lang, gettext_dir, local=False)
+
+    # switch direction for RTL languages
     if lang in ("he", "ar", "fa"):
         app.setLayoutDirection(Qt.RightToLeft)
     else:
         app.setLayoutDirection(Qt.LeftToRight)
-    # qt
+
+    # load qt translations
     _qtrans = QTranslator()
-    if _qtrans.load("qt_" + lang, ldir):
+    qt_dir = os.path.join(ldir, "qt")
+    qt_lang = lang.replace("-", "_")
+    if _qtrans.load("qtbase_" + qt_lang, qt_dir):
         app.installTranslator(_qtrans)
 
 

qt/i18n/.gitignore

@@ -1 +1,2 @@
 .build
+po

qt/i18n/build-mo-files

@@ -3,14 +3,14 @@
 # build mo files
 #
 
-targetDir="../aqt_data/locale"
+targetDir="../aqt_data/locale/gettext"
 mkdir -p $targetDir
 
 echo "Compiling *.po..."
-for file in translations/anki.pot/*
+for file in po/desktop/*/anki.po
 do
     outdir=$(echo $file | \
-        perl -pe "s%translations/anki.pot/(.*)%$targetDir/\1/LC_MESSAGES%")
+        perl -pe "s%po/desktop/(.*)/anki.po%$targetDir/\1/LC_MESSAGES%")
     outfile="$outdir/anki.mo"
     mkdir -p $outdir
     msgfmt $file --output-file=$outfile

qt/i18n/copy-qt-files

@@ -2,7 +2,7 @@
 
 set -e
 
-out=../aqt_data/locale
+out=../aqt_data/locale/qt
 mkdir -p $out
 
 qtTranslations=$(python -c "from PyQt5.QtCore import *; print(QLibraryInfo.location(QLibraryInfo.TranslationsPath))")

qt/i18n/pull-git

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ ! -d po ]; then
+  git clone https://github.com/ankitects/anki-desktop-i18n po
+fi
+
+echo "Updating translations from git..."
+(cd po && git pull)
+

qt/i18n/update-crowdin

@@ -1,20 +0,0 @@
-#!/bin/bash
-#
-# Sends the latest strings from the source code to crowdin.
-# To use this, key must be set to a crowdin API key.
-#
-
-set -e
-
-proj=anki
-
-if [ "$key" = "" ]; then
-    echo "key not defined"
-    exit 1
-fi
-
-./update-pot
-
-curl \
-  -F "files[/anki.pot]=@anki.pot" \
-  https://api.crowdin.com/api/project/$proj/update-file?key=$key

qt/i18n/update-from-crowdin

@@ -1,38 +0,0 @@
-#!/bin/bash
-#
-# Pulls the latest translations from crowdin and commits them here.
-# To use this, key must be set to a crowdin API key.
-# Aborts if there are any uncommited changes prior to running.
-#
-
-set -e
-
-proj=anki
-
-if [ "$key" = "" ]; then
-    echo "key not defined"
-    exit 1
-fi
-
-if ! git diff-index --quiet HEAD --; then
-    echo "working directory is not clean"
-    exit 1
-fi
-
-# fetch translations from crowdin
-if [ ! -f all.zip ]; then
-    curl https://api.crowdin.com/api/project/$proj/export?key=$key
-    curl -o all.zip https://api.crowdin.com/api/project/$proj/download/all.zip?key=$key
-fi
-
-# unzip
-unzip -o all.zip
-
-# make sure translations are valid
-python check-po-files.py
-
-rm all.zip
-
-# commit them to the repo
-git add translations
-git commit -m 'update translations' || true

qt/i18n/update-pot

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# update translation files
+# update template .pot file from source code strings
 #
 
 
@@ -16,5 +16,5 @@ for i in qt/aqt/{*.py,forms/*.py}; do
     echo $i >> $all
 done
 
-xgettext -cT: -s --no-wrap --files-from=$all --output=qt/i18n/anki.pot
+xgettext -cT: -s --no-wrap --files-from=$all --output=qt/i18n/po/desktop/anki.pot
 rm $all