gotext/domain.go
Lysander Trischler 2f11fcd5f2 Remove unused dependency
While figuring out what both dependencies in gotext are used for, I
quickly noticed that one is actually useless at the moment. The `tag`
field is only set but never used, so it can be dropped. It was
introduced three years ago in bb276626f3
and was even not used back then. I didn't check the history any further,
though. Removing the private field allows getting rid of a third-party
dependency entirely.
2023-09-24 22:28:12 +02:00

730 lines
17 KiB
Go

package gotext
import (
"bytes"
"encoding/gob"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"github.com/leonelquinteros/gotext/plurals"
)
// Domain has all the common functions for dealing with a gettext domain
// it's initialized with a GettextFile (which represents either a Po or Mo file)
type Domain struct {
Headers HeaderMap
// Language header
Language string
// Plural-Forms header
PluralForms string
// Preserve comments at head of PO for round-trip
headerComments []string
// Parsed Plural-Forms header values
nplurals int
plural string
pluralforms plurals.Expression
// Storage
translations map[string]*Translation
contextTranslations map[string]map[string]*Translation
pluralTranslations map[string]*Translation
// Sync Mutex
trMutex sync.RWMutex
pluralMutex sync.RWMutex
// Parsing buffers
trBuffer *Translation
ctxBuffer string
refBuffer string
}
// Preserve MIMEHeader behaviour, without the canonicalisation
type HeaderMap map[string][]string
func (m HeaderMap) Add(key, value string) {
m[key] = append(m[key], value)
}
func (m HeaderMap) Del(key string) {
delete(m, key)
}
func (m HeaderMap) Get(key string) string {
if m == nil {
return ""
}
v := m[key]
if len(v) == 0 {
return ""
}
return v[0]
}
func (m HeaderMap) Set(key, value string) {
m[key] = []string{value}
}
func (m HeaderMap) Values(key string) []string {
if m == nil {
return nil
}
return m[key]
}
func NewDomain() *Domain {
domain := new(Domain)
domain.Headers = make(HeaderMap)
domain.headerComments = make([]string, 0)
domain.translations = make(map[string]*Translation)
domain.contextTranslations = make(map[string]map[string]*Translation)
domain.pluralTranslations = make(map[string]*Translation)
return domain
}
func (do *Domain) pluralForm(n int) int {
// do we really need locking here? not sure how this plurals.Expression works, so sticking with it for now
do.pluralMutex.RLock()
defer do.pluralMutex.RUnlock()
// Failure fallback
if do.pluralforms == nil {
/* Use the Germanic plural rule. */
if n == 1 {
return 0
}
return 1
}
return do.pluralforms.Eval(uint32(n))
}
// parseHeaders retrieves data from previously parsed headers. it's called by both Mo and Po when parsing
func (do *Domain) parseHeaders() {
raw := ""
if _, ok := do.translations[raw]; ok {
raw = do.translations[raw].Get()
}
// textproto.ReadMIMEHeader() forces keys through CanonicalMIMEHeaderKey(); must read header manually to have one-to-one round-trip of keys
languageKey := "Language"
pluralFormsKey := "Plural-Forms"
rawLines := strings.Split(raw, "\n")
for _, line := range rawLines {
if len(line) == 0 {
continue
}
colonIdx := strings.Index(line, ":")
if colonIdx < 0 {
continue
}
key := line[:colonIdx]
lowerKey := strings.ToLower(key)
if lowerKey == strings.ToLower(languageKey) {
languageKey = key
} else if lowerKey == strings.ToLower(pluralFormsKey) {
pluralFormsKey = key
}
value := strings.TrimSpace(line[colonIdx+1:])
do.Headers.Add(key, value)
}
// Get/save needed headers
do.Language = do.Headers.Get(languageKey)
do.PluralForms = do.Headers.Get(pluralFormsKey)
// Parse Plural-Forms formula
if do.PluralForms == "" {
return
}
// Split plural form header value
pfs := strings.Split(do.PluralForms, ";")
// Parse values
for _, i := range pfs {
vs := strings.SplitN(i, "=", 2)
if len(vs) != 2 {
continue
}
switch strings.TrimSpace(vs[0]) {
case "nplurals":
do.nplurals, _ = strconv.Atoi(vs[1])
case "plural":
do.plural = vs[1]
if expr, err := plurals.Compile(do.plural); err == nil {
do.pluralforms = expr
}
}
}
}
// Drops any translations stored that have not been Set*() since 'po'
// was initialised
func (do *Domain) DropStaleTranslations() {
do.trMutex.Lock()
do.pluralMutex.Lock()
defer do.trMutex.Unlock()
defer do.pluralMutex.Unlock()
for name, ctx := range do.contextTranslations {
for id, trans := range ctx {
if trans.IsStale() {
delete(ctx, id)
}
}
if len(ctx) == 0 {
delete(do.contextTranslations, name)
}
}
for id, trans := range do.translations {
if trans.IsStale() {
delete(do.translations, id)
}
}
}
// Set source references for a given translation
func (do *Domain) SetRefs(str string, refs []string) {
do.trMutex.Lock()
do.pluralMutex.Lock()
defer do.trMutex.Unlock()
defer do.pluralMutex.Unlock()
if trans, ok := do.translations[str]; ok {
trans.Refs = refs
} else {
trans = NewTranslation()
trans.ID = str
trans.SetRefs(refs)
do.translations[str] = trans
}
}
// Get source references for a given translation
func (do *Domain) GetRefs(str string) []string {
// Sync read
do.trMutex.RLock()
defer do.trMutex.RUnlock()
if do.translations != nil {
if trans, ok := do.translations[str]; ok {
return trans.Refs
}
}
return nil
}
// Set the translation of a given string
func (do *Domain) Set(id, str string) {
do.trMutex.Lock()
do.pluralMutex.Lock()
defer do.trMutex.Unlock()
defer do.pluralMutex.Unlock()
if trans, ok := do.translations[id]; ok {
trans.Set(str)
} else {
trans = NewTranslation()
trans.ID = id
trans.Set(str)
do.translations[str] = trans
}
}
func (do *Domain) Get(str string, vars ...interface{}) string {
// Sync read
do.trMutex.RLock()
defer do.trMutex.RUnlock()
if do.translations != nil {
if _, ok := do.translations[str]; ok {
return Printf(do.translations[str].Get(), vars...)
}
}
// Return the same we received by default
return Printf(str, vars...)
}
// Set the (N)th plural form for the given string
func (do *Domain) SetN(id, plural string, n int, str string) {
// Get plural form _before_ lock down
pluralForm := do.pluralForm(n)
do.trMutex.Lock()
do.pluralMutex.Lock()
defer do.trMutex.Unlock()
defer do.pluralMutex.Unlock()
if trans, ok := do.translations[id]; ok {
trans.SetN(pluralForm, str)
} else {
trans = NewTranslation()
trans.ID = id
trans.PluralID = plural
trans.SetN(pluralForm, str)
do.translations[str] = trans
}
}
// GetN retrieves the (N)th plural form of Translation for the given string.
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) GetN(str, plural string, n int, vars ...interface{}) string {
// Sync read
do.trMutex.RLock()
defer do.trMutex.RUnlock()
if do.translations != nil {
if _, ok := do.translations[str]; ok {
return Printf(do.translations[str].GetN(do.pluralForm(n)), vars...)
}
}
// Parse plural forms to distinguish between plural and singular
if do.pluralForm(n) == 0 {
return Printf(str, vars...)
}
return Printf(plural, vars...)
}
// Set the translation for the given string in the given context
func (do *Domain) SetC(id, ctx, str string) {
do.trMutex.Lock()
do.pluralMutex.Lock()
defer do.trMutex.Unlock()
defer do.pluralMutex.Unlock()
if context, ok := do.contextTranslations[ctx]; ok {
if trans, hasTrans := context[id]; hasTrans {
trans.Set(str)
} else {
trans = NewTranslation()
trans.ID = id
trans.Set(str)
context[id] = trans
}
} else {
trans := NewTranslation()
trans.ID = id
trans.Set(str)
do.contextTranslations[ctx] = map[string]*Translation{
id: trans,
}
}
}
// GetC retrieves the corresponding Translation for a given string in the given context.
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) GetC(str, ctx string, vars ...interface{}) string {
do.trMutex.RLock()
defer do.trMutex.RUnlock()
if do.contextTranslations != nil {
if _, ok := do.contextTranslations[ctx]; ok {
if do.contextTranslations[ctx] != nil {
if _, ok := do.contextTranslations[ctx][str]; ok {
return Printf(do.contextTranslations[ctx][str].Get(), vars...)
}
}
}
}
// Return the string we received by default
return Printf(str, vars...)
}
// Set the (N)th plural form for the given string in the given context
func (do *Domain) SetNC(id, plural, ctx string, n int, str string) {
// Get plural form _before_ lock down
pluralForm := do.pluralForm(n)
do.trMutex.Lock()
do.pluralMutex.Lock()
defer do.trMutex.Unlock()
defer do.pluralMutex.Unlock()
if context, ok := do.contextTranslations[ctx]; ok {
if trans, hasTrans := context[id]; hasTrans {
trans.SetN(pluralForm, str)
} else {
trans = NewTranslation()
trans.ID = id
trans.SetN(pluralForm, str)
context[id] = trans
}
} else {
trans := NewTranslation()
trans.ID = id
trans.SetN(pluralForm, str)
do.contextTranslations[ctx] = map[string]*Translation{
id: trans,
}
}
}
// GetNC retrieves the (N)th plural form of Translation for the given string in the given context.
// Supports optional parameters (vars... interface{}) to be inserted on the formatted string using the fmt.Printf syntax.
func (do *Domain) GetNC(str, plural string, n int, ctx string, vars ...interface{}) string {
do.trMutex.RLock()
defer do.trMutex.RUnlock()
if do.contextTranslations != nil {
if _, ok := do.contextTranslations[ctx]; ok {
if do.contextTranslations[ctx] != nil {
if _, ok := do.contextTranslations[ctx][str]; ok {
return Printf(do.contextTranslations[ctx][str].GetN(do.pluralForm(n)), vars...)
}
}
}
}
if n == 1 {
return Printf(str, vars...)
}
return Printf(plural, vars...)
}
// IsTranslated reports whether a string is translated
func (do *Domain) IsTranslated(str string) bool {
return do.IsTranslatedN(str, 0)
}
// IsTranslatedN reports whether a plural string is translated
func (do *Domain) IsTranslatedN(str string, n int) bool {
do.trMutex.RLock()
defer do.trMutex.RUnlock()
if do.translations == nil {
return false
}
tr, ok := do.translations[str]
if !ok {
return false
}
return tr.IsTranslatedN(n)
}
// IsTranslatedC reports whether a context string is translated
func (do *Domain) IsTranslatedC(str, ctx string) bool {
return do.IsTranslatedNC(str, 0, ctx)
}
// IsTranslatedNC reports whether a plural context string is translated
func (do *Domain) IsTranslatedNC(str string, n int, ctx string) bool {
do.trMutex.RLock()
defer do.trMutex.RUnlock()
if do.contextTranslations == nil {
return false
}
translations, ok := do.contextTranslations[ctx]
if !ok {
return false
}
tr, ok := translations[str]
if !ok {
return false
}
return tr.IsTranslatedN(n)
}
// GetTranslations returns a copy of every translation in the domain. It does not support contexts.
func (do *Domain) GetTranslations() map[string]*Translation {
all := make(map[string]*Translation, len(do.translations))
do.trMutex.RLock()
defer do.trMutex.RUnlock()
for msgID, trans := range do.translations {
newTrans := NewTranslation()
newTrans.ID = trans.ID
newTrans.PluralID = trans.PluralID
newTrans.dirty = trans.dirty
if len(trans.Refs) > 0 {
newTrans.Refs = make([]string, len(trans.Refs))
copy(newTrans.Refs, trans.Refs)
}
for k, v := range trans.Trs {
newTrans.Trs[k] = v
}
all[msgID] = newTrans
}
return all
}
type SourceReference struct {
path string
line int
context string
trans *Translation
}
func extractPathAndLine(ref string) (string, int) {
var path string
var line int
colonIdx := strings.IndexRune(ref, ':')
if colonIdx >= 0 {
path = ref[:colonIdx]
line, _ = strconv.Atoi(ref[colonIdx+1:])
} else {
path = ref
line = 0
}
return path, line
}
// MarshalText implements encoding.TextMarshaler interface
// Assists round-trip of POT/PO content
func (do *Domain) MarshalText() ([]byte, error) {
var buf bytes.Buffer
if len(do.headerComments) > 0 {
buf.WriteString(strings.Join(do.headerComments, "\n"))
buf.WriteByte(byte('\n'))
}
buf.WriteString("msgid \"\"\nmsgstr \"\"")
// Standard order consistent with xgettext
headerOrder := map[string]int{
"project-id-version": 0,
"report-msgid-bugs-to": 1,
"pot-creation-date": 2,
"po-revision-date": 3,
"last-translator": 4,
"language-team": 5,
"language": 6,
"mime-version": 7,
"content-type": 9,
"content-transfer-encoding": 10,
"plural-forms": 11,
}
headerKeys := make([]string, 0, len(do.Headers))
for k, _ := range do.Headers {
headerKeys = append(headerKeys, k)
}
sort.Slice(headerKeys, func(i, j int) bool {
var iOrder int
var jOrder int
var ok bool
if iOrder, ok = headerOrder[strings.ToLower(headerKeys[i])]; !ok {
iOrder = 8
}
if jOrder, ok = headerOrder[strings.ToLower(headerKeys[j])]; !ok {
jOrder = 8
}
if iOrder < jOrder {
return true
}
if iOrder > jOrder {
return false
}
return headerKeys[i] < headerKeys[j]
})
for _, k := range headerKeys {
// Access Headers map directly so as not to canonicalise
v := do.Headers[k]
for _, value := range v {
buf.WriteString("\n\"" + k + ": " + value + "\\n\"")
}
}
// Just as with headers, output translations in consistent order (to minimise diffs between round-trips), with (first) source reference taking priority, followed by context and finally ID
references := make([]SourceReference, 0)
for name, ctx := range do.contextTranslations {
for id, trans := range ctx {
if id == "" {
continue
}
if len(trans.Refs) > 0 {
path, line := extractPathAndLine(trans.Refs[0])
references = append(references, SourceReference{
path,
line,
name,
trans,
})
} else {
references = append(references, SourceReference{
"",
0,
name,
trans,
})
}
}
}
for id, trans := range do.translations {
if id == "" {
continue
}
if len(trans.Refs) > 0 {
path, line := extractPathAndLine(trans.Refs[0])
references = append(references, SourceReference{
path,
line,
"",
trans,
})
} else {
references = append(references, SourceReference{
"",
0,
"",
trans,
})
}
}
sort.Slice(references, func(i, j int) bool {
if references[i].path < references[j].path {
return true
}
if references[i].path > references[j].path {
return false
}
if references[i].line < references[j].line {
return true
}
if references[i].line > references[j].line {
return false
}
if references[i].context < references[j].context {
return true
}
if references[i].context > references[j].context {
return false
}
return references[i].trans.ID < references[j].trans.ID
})
for _, ref := range references {
trans := ref.trans
if len(trans.Refs) > 0 {
buf.WriteString("\n\n#: " + strings.Join(trans.Refs, " "))
} else {
buf.WriteByte(byte('\n'))
}
if ref.context == "" {
buf.WriteString("\nmsgid \"" + EscapeSpecialCharacters(trans.ID) + "\"")
} else {
buf.WriteString("\nmsgctxt \"" + EscapeSpecialCharacters(ref.context) + "\"\nmsgid \"" + EscapeSpecialCharacters(trans.ID) + "\"")
}
if trans.PluralID == "" {
buf.WriteString("\nmsgstr \"" + EscapeSpecialCharacters(trans.Trs[0]) + "\"")
} else {
buf.WriteString("\nmsgid_plural \"" + trans.PluralID + "\"")
for i, tr := range trans.Trs {
buf.WriteString("\nmsgstr[" + EscapeSpecialCharacters(strconv.Itoa(i)) + "] \"" + tr + "\"")
}
}
}
return buf.Bytes(), nil
}
func EscapeSpecialCharacters(s string) string {
s = regexp.MustCompile(`([^\\])(")`).ReplaceAllString(s, "$1\\\"") // Escape non-escaped double quotation marks
if strings.Count(s, "\n") == 0 {
return s
}
// Handle EOL and multi-lines
// Only one line, but finishing with \n
if strings.Count(s, "\n") == 1 && strings.HasSuffix(s, "\n") {
return strings.ReplaceAll(s, "\n", "\\n")
}
elems := strings.Split(s, "\n")
// Skip last element for multiline which is an empty
var shouldEndWithEOL bool
if elems[len(elems)-1] == "" {
elems = elems[:len(elems)-1]
shouldEndWithEOL = true
}
data := []string{(`"`)}
for i, v := range elems {
l := fmt.Sprintf(`"%s\n"`, v)
// Last element without EOL
if i == len(elems)-1 && !shouldEndWithEOL {
l = fmt.Sprintf(`"%s"`, v)
}
// Remove finale " to last element as the whole string will be quoted
if i == len(elems)-1 {
l = strings.TrimSuffix(l, `"`)
}
data = append(data, l)
}
return strings.Join(data, "\n")
}
// MarshalBinary implements encoding.BinaryMarshaler interface
func (do *Domain) MarshalBinary() ([]byte, error) {
obj := new(TranslatorEncoding)
obj.Headers = do.Headers
obj.Language = do.Language
obj.PluralForms = do.PluralForms
obj.Nplurals = do.nplurals
obj.Plural = do.plural
obj.Translations = do.translations
obj.Contexts = do.contextTranslations
var buff bytes.Buffer
encoder := gob.NewEncoder(&buff)
err := encoder.Encode(obj)
return buff.Bytes(), err
}
// UnmarshalBinary implements encoding.BinaryUnmarshaler interface
func (do *Domain) UnmarshalBinary(data []byte) error {
buff := bytes.NewBuffer(data)
obj := new(TranslatorEncoding)
decoder := gob.NewDecoder(buff)
err := decoder.Decode(obj)
if err != nil {
return err
}
do.Headers = obj.Headers
do.Language = obj.Language
do.PluralForms = obj.PluralForms
do.nplurals = obj.Nplurals
do.plural = obj.Plural
do.translations = obj.Translations
do.contextTranslations = obj.Contexts
if expr, err := plurals.Compile(do.plural); err == nil {
do.pluralforms = expr
}
return nil
}