Add support for multiple languages

gettext supports supplying multiple languages using the colon character
$ LANGUAGE=pt_BR:pt_PT:en_US foo bar
which are used as fallback: when a translation for the first one is not
available, the second language is used. If no language contains
a translation for the string `msgid` is returned.

This patch adds this support into gotext.

1/ config struct
 - 'language' was renamed to 'languages' and is a slice of strings
 - 'storage' was renamed to 'locales' and is a slice of Locale pointers

2/ loadStorage()
 - all loaded languages are iterated over

3/ GetLanguages()
 - new function returns the languages from the config
 - GetLanguage() uses the first element of it, keeping the compatibility

4/ SetLanguage(), Configure()
 - the language string is split at colon and iterated over

5/ Get*()
 - languages are iterated and the first translation for given string is
   returned

6/ IsTranslated*()
 - new optional parameter (langs) has been added

7/ Locale.GetActualLanguage()
 - it checks the filesystem and determines what the actual language code
   is: for 'cs_CZ', just 'cs' may be returned, depending on the actual
   name of the .mo/.po file.

8/ GetLocales/GetStorage, SetLocales/SetStorage
 - Following recent changes, created public functions to manipulate with
   global configuration's locales. The *Storage functions are renamed
   in later commit to reduce the amount of changes in one commit.
This commit is contained in:
Matyas Horky 2023-04-17 14:15:55 +02:00 committed by mhorky
parent 12a99145a3
commit 4ec5f399f4
3 changed files with 259 additions and 101 deletions

306
gotext.go
View file

@ -24,6 +24,7 @@ package gotext
import (
"encoding/gob"
"strings"
"sync"
)
@ -31,47 +32,56 @@ import (
type config struct {
sync.RWMutex
// Path to library directory where all locale directories and Translation files are.
library string
// Default domain to look at when no domain is specified. Used by package level functions.
domain string
// Language set.
language string
// Path to library directory where all locale directories and Translation files are.
library string
languages []string
// Storage for package level methods
storage *Locale
locales []*Locale
}
var globalConfig *config
var FallbackLocale = "en_US"
func init() {
// Init default configuration
globalConfig = &config{
domain: "default",
language: "en_US",
library: "/usr/local/share/locale",
storage: nil,
domain: "default",
languages: []string{FallbackLocale},
library: "/usr/local/share/locale",
locales: nil,
}
// Register Translator types for gob encoding
gob.Register(TranslatorEncoding{})
}
// loadStorage creates a new Locale object at package level based on the Global variables settings.
// loadStorage creates a new Locale object at package level based on the Global variables settings
// for every language specified using Configure.
// It's called automatically when trying to use Get or GetD methods.
func loadStorage(force bool) {
globalConfig.Lock()
if globalConfig.storage == nil || force {
globalConfig.storage = NewLocale(globalConfig.library, globalConfig.language)
if globalConfig.locales == nil || force {
var locales []*Locale
for _, language := range globalConfig.languages {
locales = append(locales, NewLocale(globalConfig.library, language))
}
globalConfig.locales = locales
}
if _, ok := globalConfig.storage.Domains[globalConfig.domain]; !ok || force {
globalConfig.storage.AddDomain(globalConfig.domain)
for _, locale := range globalConfig.locales {
if _, ok := locale.Domains[globalConfig.domain]; !ok || force {
locale.AddDomain(globalConfig.domain)
}
locale.SetDomain(globalConfig.domain)
}
globalConfig.storage.SetDomain(globalConfig.domain)
globalConfig.Unlock()
}
@ -80,8 +90,9 @@ func loadStorage(force bool) {
func GetDomain() string {
var dom string
globalConfig.RLock()
if globalConfig.storage != nil {
dom = globalConfig.storage.GetDomain()
if globalConfig.locales != nil {
// All locales have the same domain
dom = globalConfig.locales[0].GetDomain()
}
if dom == "" {
dom = globalConfig.domain
@ -96,28 +107,43 @@ func GetDomain() string {
func SetDomain(dom string) {
globalConfig.Lock()
globalConfig.domain = dom
if globalConfig.storage != nil {
globalConfig.storage.SetDomain(dom)
if globalConfig.locales != nil {
for _, locale := range globalConfig.locales {
locale.SetDomain(dom)
}
}
globalConfig.Unlock()
loadStorage(true)
}
// GetLanguage is the language getter for the package configuration
// GetLanguage returns the language gotext will translate into.
// If multiple languages have been supplied, the first one will be returned.
// If no language has been supplied, the fallback will be returned.
func GetLanguage() string {
globalConfig.RLock()
lang := globalConfig.language
globalConfig.RUnlock()
return lang
languages := GetLanguages()
if len(languages) == 0 {
return FallbackLocale
}
return languages[0]
}
// SetLanguage sets the language code to be used at package level.
// GetLanguages returns all languages that have been supplied.
func GetLanguages() []string {
globalConfig.RLock()
defer globalConfig.RUnlock()
return globalConfig.languages
}
// SetLanguage sets the language code (or colon separated language codes) to be used at package level.
// It reloads the corresponding Translation file.
func SetLanguage(lang string) {
globalConfig.Lock()
globalConfig.language = SimplifiedLocale(lang)
var languages []string
for _, language := range strings.Split(lang, ":") {
languages = append(languages, SimplifiedLocale(language))
}
globalConfig.languages = languages
globalConfig.Unlock()
loadStorage(true)
@ -142,28 +168,53 @@ func SetLibrary(lib string) {
loadStorage(true)
}
// GetStorage is the locale storage getter for the package configuration.
func GetStorage() *Locale {
func GetLocales() []*Locale {
globalConfig.RLock()
storage := globalConfig.storage
globalConfig.RUnlock()
return storage
defer globalConfig.RUnlock()
return globalConfig.locales
}
// SetStorage allows overridding the global Locale object with one built manually with NewLocale().
// GetStorage is the locale storage getter for the package configuration.
//
// It returns the first locale returned by GetLocales.
// This function exists for backwards compatibility, prefer using GetLocales directly.
func GetStorage() *Locale {
locales := GetLocales()
if len(locales) > 0 {
return GetLocales()[0]
}
return nil
}
// SetLocales allows for overriding the global Locale objects with ones built manually with
// NewLocale(). This makes it possible to attach custom Domain objects from in-memory po/mo.
// The library, language and domain of the first Locale will set the default global configuration.
func SetLocales(locales []*Locale) {
globalConfig.Lock()
defer globalConfig.Unlock()
globalConfig.locales = locales
globalConfig.library = locales[0].path
globalConfig.domain = locales[0].defaultDomain
var languages []string
for _, locale := range locales {
languages = append(languages, locale.lang)
}
globalConfig.languages = languages
}
// SetStorage allows overriding the global Locale object with one built manually with NewLocale().
// This allows then to attach to the locale Domains object in memory po or mo files (embedded or in any directory),
// for each domain.
// Locale library, language and domain properties will apply on default global configuration.
// Any domain not loaded yet will use to the just in time domain loading process.
// Note that any call to gotext.Set* or Configure will invalidate this override.
//
// This works by calling SetLocales with just one Locale object.
// This function exists for backwards compatibility, prefer using SetLocales directly.
func SetStorage(storage *Locale) {
globalConfig.Lock()
globalConfig.storage = storage
globalConfig.library = storage.path
globalConfig.language = storage.lang
globalConfig.domain = storage.defaultDomain
globalConfig.Unlock()
SetLocales([]*Locale{storage})
}
// Configure sets all configuration variables to be used at package level and reloads the corresponding Translation file.
@ -173,7 +224,11 @@ func SetStorage(storage *Locale) {
func Configure(lib, lang, dom string) {
globalConfig.Lock()
globalConfig.library = lib
globalConfig.language = SimplifiedLocale(lang)
var languages []string
for _, language := range strings.Split(lang, ":") {
languages = append(languages, SimplifiedLocale(language))
}
globalConfig.languages = languages
globalConfig.domain = dom
globalConfig.Unlock()
@ -198,16 +253,20 @@ func GetD(dom, str string, vars ...interface{}) string {
// Try to load default package Locale storage
loadStorage(false)
// Return Translation
globalConfig.RLock()
defer globalConfig.RUnlock()
if _, ok := globalConfig.storage.Domains[dom]; !ok {
globalConfig.storage.AddDomain(dom)
var tr string
for i, locale := range globalConfig.locales {
if _, ok := locale.Domains[dom]; !ok {
locale.AddDomain(dom)
}
if !locale.IsTranslatedD(dom, str) && i < (len(globalConfig.locales)-1) {
continue
}
tr = locale.GetD(dom, str, vars...)
break
}
tr := globalConfig.storage.GetD(dom, str, vars...)
globalConfig.RUnlock()
return tr
}
@ -217,16 +276,20 @@ func GetND(dom, str, plural string, n int, vars ...interface{}) string {
// Try to load default package Locale storage
loadStorage(false)
// Return Translation
globalConfig.RLock()
defer globalConfig.RUnlock()
if _, ok := globalConfig.storage.Domains[dom]; !ok {
globalConfig.storage.AddDomain(dom)
var tr string
for i, locale := range globalConfig.locales {
if _, ok := locale.Domains[dom]; !ok {
locale.AddDomain(dom)
}
if !locale.IsTranslatedND(dom, str, n) && i < (len(globalConfig.locales)-1) {
continue
}
tr = locale.GetND(dom, str, plural, n, vars...)
break
}
tr := globalConfig.storage.GetND(dom, str, plural, n, vars...)
globalConfig.RUnlock()
return tr
}
@ -248,11 +311,17 @@ func GetDC(dom, str, ctx string, vars ...interface{}) string {
// Try to load default package Locale storage
loadStorage(false)
// Return Translation
globalConfig.RLock()
tr := globalConfig.storage.GetDC(dom, str, ctx, vars...)
globalConfig.RUnlock()
defer globalConfig.RUnlock()
var tr string
for i, locale := range globalConfig.locales {
if !locale.IsTranslatedDC(dom, str, ctx) && i < (len(globalConfig.locales)-1) {
continue
}
tr = locale.GetDC(dom, str, ctx, vars...)
break
}
return tr
}
@ -264,62 +333,101 @@ func GetNDC(dom, str, plural string, n int, ctx string, vars ...interface{}) str
// Return Translation
globalConfig.RLock()
tr := globalConfig.storage.GetNDC(dom, str, plural, n, ctx, vars...)
globalConfig.RUnlock()
defer globalConfig.RUnlock()
var tr string
for i, locale := range globalConfig.locales {
if !locale.IsTranslatedNDC(dom, str, n, ctx) && i < (len(globalConfig.locales)-1) {
continue
}
tr = locale.GetNDC(dom, str, plural, n, ctx, vars...)
break
}
return tr
}
// IsTranslated reports whether a string is translated
func IsTranslated(str string) bool {
return IsTranslatedND(GetDomain(), str, 0)
// IsTranslated reports whether a string is translated in given languages.
// When the langs argument is omitted, the output of GetLanguages is used.
func IsTranslated(str string, langs ...string) bool {
return IsTranslatedND(GetDomain(), str, 0, langs...)
}
// IsTranslatedN reports whether a plural string is translated
func IsTranslatedN(str string, n int) bool {
return IsTranslatedND(GetDomain(), str, n)
// IsTranslatedN reports whether a plural string is translated in given languages.
// When the langs argument is omitted, the output of GetLanguages is used.
func IsTranslatedN(str string, n int, langs ...string) bool {
return IsTranslatedND(GetDomain(), str, n, langs...)
}
// IsTranslatedD reports whether a domain string is translated
func IsTranslatedD(dom, str string) bool {
return IsTranslatedND(dom, str, 0)
// IsTranslatedD reports whether a domain string is translated in given languages.
// When the langs argument is omitted, the output of GetLanguages is used.
func IsTranslatedD(dom, str string, langs ...string) bool {
return IsTranslatedND(dom, str, 0, langs...)
}
// IsTranslatedND reports whether a plural domain string is translated
func IsTranslatedND(dom, str string, n int) bool {
loadStorage(false)
globalConfig.RLock()
defer globalConfig.RUnlock()
if _, ok := globalConfig.storage.Domains[dom]; !ok {
globalConfig.storage.AddDomain(dom)
// IsTranslatedND reports whether a plural domain string is translated in any of given languages.
// When the langs argument is omitted, the output of GetLanguages is used.
func IsTranslatedND(dom, str string, n int, langs ...string) bool {
if len(langs) == 0 {
langs = GetLanguages()
}
return globalConfig.storage.IsTranslatedND(dom, str, n)
}
// IsTranslatedC reports whether a context string is translated
func IsTranslatedC(str, ctx string) bool {
return IsTranslatedNDC(GetDomain(), str, 0, ctx)
}
// IsTranslatedNC reports whether a plural context string is translated
func IsTranslatedNC(str string, n int, ctx string) bool {
return IsTranslatedNDC(GetDomain(), str, n, ctx)
}
// IsTranslatedDC reports whether a domain context string is translated
func IsTranslatedDC(dom, str, ctx string) bool {
return IsTranslatedNDC(dom, str, 0, ctx)
}
// IsTranslatedNDC reports whether a plural domain context string is translated
func IsTranslatedNDC(dom, str string, n int, ctx string) bool {
loadStorage(false)
globalConfig.RLock()
defer globalConfig.RUnlock()
return globalConfig.storage.IsTranslatedNDC(dom, str, n, ctx)
for _, lang := range langs {
lang = SimplifiedLocale(lang)
for _, supportedLocale := range globalConfig.locales {
if lang != supportedLocale.GetActualLanguage(dom) {
continue
}
return supportedLocale.IsTranslatedND(dom, str, n)
}
}
return false
}
// IsTranslatedC reports whether a context string is translated in given languages.
// When the langs argument is omitted, the output of GetLanguages is used.
func IsTranslatedC(str, ctx string, langs ...string) bool {
return IsTranslatedNDC(GetDomain(), str, 0, ctx, langs...)
}
// IsTranslatedNC reports whether a plural context string is translated in given languages.
// When the langs argument is omitted, the output of GetLanguages is used.
func IsTranslatedNC(str string, n int, ctx string, langs ...string) bool {
return IsTranslatedNDC(GetDomain(), str, n, ctx, langs...)
}
// IsTranslatedDC reports whether a domain context string is translated in given languages.
// When the langs argument is omitted, the output of GetLanguages is used.
func IsTranslatedDC(dom, str, ctx string, langs ...string) bool {
return IsTranslatedNDC(dom, str, 0, ctx, langs...)
}
// IsTranslatedNDC reports whether a plural domain context string is translated in any of given languages.
// When the langs argument is omitted, the output of GetLanguages is used.
func IsTranslatedNDC(dom, str string, n int, ctx string, langs ...string) bool {
if len(langs) == 0 {
langs = GetLanguages()
}
loadStorage(false)
globalConfig.RLock()
defer globalConfig.RUnlock()
for _, lang := range langs {
lang = SimplifiedLocale(lang)
for _, locale := range globalConfig.locales {
if lang != locale.GetActualLanguage(dom) {
continue
}
return locale.IsTranslatedNDC(dom, str, n, ctx)
}
}
return false
}

View file

@ -186,13 +186,29 @@ msgstr "Another text on another domain"
t.Errorf("Expected 'Another text on another domain' but got '%s'", tr)
}
// Test IsTranslation functions
// IsTranslated tests for when the string is translated in English.
if !IsTranslated("My text") {
t.Error("'My text' should be reported as translated.")
t.Error("'My text' should be reported as translated when 'langs' is omitted.")
}
if !IsTranslated("My text", "en_US") {
t.Error("'My text' should be reported as translated when 'langs' is 'en_US'.")
}
if IsTranslated("My text", "cs_CZ") {
t.Error("'My text' should be reported as not translated when 'langs' is 'cs_CZ'.")
}
if !IsTranslated("My text", "en_US", "cs_CZ") {
t.Error("'My text' should be reported as translated when 'langs' is 'en_US, cs_CZ'.")
}
// IsTranslated tests for when the string is not translated in English
if IsTranslated("Another string") {
t.Error("'Another string' should be reported as not translated.")
}
if IsTranslated("String not in .po") {
t.Error("'String not in .po' should be reported as not translated.")
}
// IsTranslated tests for plurals and contexts
plural := "One with var: %s"
if !IsTranslated(plural) {
t.Errorf("'%s' should be reported as translated for singular.", plural)

View file

@ -111,6 +111,40 @@ func (l *Locale) findExt(dom, ext string) string {
return ""
}
// GetActualLanguage inspects the filesystem and decides whether to strip
// a CC part of the ll_CC locale string.
func (l *Locale) GetActualLanguage(dom string) string {
extensions := []string{"mo", "po"}
var fp string
for _, ext := range extensions {
// 'll' (or 'll_CC') exists, and it was specified as-is
fp = path.Join(l.path, l.lang, "LC_MESSAGES", dom+"."+ext)
if l.fileExists(fp) {
return l.lang
}
// 'll' exists, but 'll_CC' was specified
if len(l.lang) > 2 {
fp = path.Join(l.path, l.lang[:2], "LC_MESSAGES", dom+"."+ext)
if l.fileExists(fp) {
return l.lang[:2]
}
}
// 'll' (or 'll_CC') exists outside of LC_category, and it was specified as-is
fp = path.Join(l.path, l.lang, dom+"."+ext)
if l.fileExists(fp) {
return l.lang
}
// 'll' exists outside of LC_category, but 'll_CC' was specified
if len(l.lang) > 2 {
fp = path.Join(l.path, l.lang[:2], dom+"."+ext)
if l.fileExists(fp) {
return l.lang[:2]
}
}
}
return ""
}
func (l *Locale) fileExists(filename string) bool {
if l.fs != nil {
_, err := fs.Stat(l.fs, filename)