From 1fc8dec04d161fc0b4007e808a48620c537b6a6e Mon Sep 17 00:00:00 2001 From: Leonel Quinteros Date: Fri, 8 Sep 2017 18:08:56 -0300 Subject: [PATCH] Rewrite PO headers parsing and handling. Implement correct GNU gettext headers format. Fix tests. Fixes #10 --- gotext_test.go | 48 +++++++++--------- locale_test.go | 6 +-- po.go | 135 +++++++++++++++++++++++++++---------------------- po_test.go | 86 +++++++++++++++++++++++-------- 4 files changed, 166 insertions(+), 109 deletions(-) diff --git a/gotext_test.go b/gotext_test.go index 58c241f..a9d87a6 100644 --- a/gotext_test.go +++ b/gotext_test.go @@ -3,6 +3,7 @@ package gotext import ( "os" "path" + "sync" "testing" ) @@ -32,10 +33,12 @@ func TestGettersSetters(t *testing.T) { func TestPackageFunctions(t *testing.T) { // Set PO content str := ` -# msgid "" -# msgstr "" +msgid "" +msgstr "Project-Id-Version: %s\n" + "Report-Msgid-Bugs-To: %s\n" + # Initial comment -# Headers below +# More Headers below "Language: en\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" @@ -55,14 +58,6 @@ msgstr[0] "This one is the singular: %s" msgstr[1] "This one is the plural: %s" msgstr[2] "And this is the second plural form: %s" -msgid "This one has invalid syntax translations" -msgid_plural "Plural index" -msgstr[abc] "Wrong index" -msgstr[1 "Forgot to close brackets" -msgstr[0] "Badly formatted string' - -msgid "Invalid formatted id[] with no translations - msgctxt "Ctx" msgid "One with var: %s" msgid_plural "Several with vars: %s" @@ -149,8 +144,8 @@ msgstr[1] "" func TestUntranslated(t *testing.T) { // Set PO content str := ` -# msgid "" -# msgstr "" +msgid "" +msgstr "" # Initial comment # Headers below "Language: en\n" @@ -258,22 +253,29 @@ msgstr[2] "And this is the second plural form: %s" t.Fatalf("Can't write to test file: %s", err.Error()) } - // Init sync channels - c1 := make(chan bool) - c2 := make(chan bool) + var wg sync.WaitGroup for i := 0; i < 100; i++ { + wg.Add(1) // Test translations - go func(done chan bool) { - Get("My text") - done <- true - }(c1) + go func() { + defer wg.Done() - go func(done chan bool) { Get("My text") - done <- true - }(c2) + GetN("One with var: %s", "Several with vars: %s", 0, "test") + }() + + wg.Add(1) + go func() { + defer wg.Done() + + Get("My text") + GetN("One with var: %s", "Several with vars: %s", 1, "test") + }() Get("My text") + GetN("One with var: %s", "Several with vars: %s", 2, "test") } + + wg.Wait() } diff --git a/locale_test.go b/locale_test.go index 2c9eb0a..8309ea2 100644 --- a/locale_test.go +++ b/locale_test.go @@ -9,8 +9,8 @@ import ( func TestLocale(t *testing.T) { // Set PO content str := ` -# msgid "" -# msgstr "" +msgid "" +msgstr "" # Initial comment # Headers below "Language: en\n" @@ -38,8 +38,6 @@ msgstr[abc] "Wrong index" msgstr[1 "Forgot to close brackets" msgstr[0] "Badly formatted string' -msgid "Invalid formatted id[] with no translations - msgctxt "Ctx" msgid "One with var: %s" msgid_plural "Several with vars: %s" diff --git a/po.go b/po.go index 7b76af5..858119a 100644 --- a/po.go +++ b/po.go @@ -79,8 +79,8 @@ Example: */ type Po struct { - // Headers - RawHeaders string + // Headers storage + Headers textproto.MIMEHeader // Language header Language string @@ -140,7 +140,6 @@ func (po *Po) ParseFile(f string) { func (po *Po) Parse(str string) { // Lock while parsing po.Lock() - defer po.Unlock() // Init storage if po.translations == nil { @@ -195,7 +194,7 @@ func (po *Po) Parse(str string) { // Multi line strings and headers if strings.HasPrefix(l, "\"") && strings.HasSuffix(l, "\"") { - state = po.parseString(l, state) + po.parseString(l, state) continue } } @@ -203,6 +202,9 @@ func (po *Po) Parse(str string) { // Save last translation buffer. po.saveBuffer() + // Unlock to parse headers + po.Unlock() + // Parse headers po.parseHeaders() } @@ -210,23 +212,24 @@ func (po *Po) Parse(str string) { // saveBuffer takes the context and translation buffers // and saves it on the translations collection func (po *Po) saveBuffer() { - // If we have something to save... - if po.trBuffer.id != "" { - // With no context... - if po.ctxBuffer == "" { - po.translations[po.trBuffer.id] = po.trBuffer - } else { - // With context... - if _, ok := po.contexts[po.ctxBuffer]; !ok { - po.contexts[po.ctxBuffer] = make(map[string]*translation) - } - po.contexts[po.ctxBuffer][po.trBuffer.id] = po.trBuffer + // With no context... + if po.ctxBuffer == "" { + po.translations[po.trBuffer.id] = po.trBuffer + } else { + // With context... + if _, ok := po.contexts[po.ctxBuffer]; !ok { + po.contexts[po.ctxBuffer] = make(map[string]*translation) } + po.contexts[po.ctxBuffer][po.trBuffer.id] = po.trBuffer - // Flush buffer - po.trBuffer = newTranslation() - po.ctxBuffer = "" + // Cleanup current context buffer if needed + if po.trBuffer.id != "" { + po.ctxBuffer = "" + } } + + // Flush translation buffer + po.trBuffer = newTranslation() } // parseContext takes a line starting with "msgctxt", @@ -286,70 +289,72 @@ func (po *Po) parseMessage(l string) { // parseString takes a well formatted string without prefix // and creates headers or attach multi-line strings when corresponding -func (po *Po) parseString(l string, state parseState) parseState { +func (po *Po) parseString(l string, state parseState) { + clean, _ := strconv.Unquote(l) + switch state { case msgStr: - // Check for multiline from previously set msgid - if po.trBuffer.id != "" { - // Append to last translation found - uq, _ := strconv.Unquote(l) - po.trBuffer.trs[len(po.trBuffer.trs)-1] += uq + // Append to last translation found + po.trBuffer.trs[len(po.trBuffer.trs)-1] += clean - } case msgID: // Multiline msgid - Append to current id - uq, _ := strconv.Unquote(l) - po.trBuffer.id += uq + po.trBuffer.id += clean + case msgIDPlural: // Multiline msgid - Append to current id - uq, _ := strconv.Unquote(l) - po.trBuffer.pluralID += uq + po.trBuffer.pluralID += clean + case msgCtxt: // Multiline context - Append to current context - ctxt, _ := strconv.Unquote(l) - po.ctxBuffer += ctxt - default: - // Otherwise is a header - h, _ := strconv.Unquote(strings.TrimSpace(l)) - po.RawHeaders += h - return head - } + po.ctxBuffer += clean - return state + } } // isValidLine checks for line prefixes to detect valid syntax. func (po *Po) isValidLine(l string) bool { - // Skip empty lines - if l == "" { - return false - } - // Check prefix - if !strings.HasPrefix(l, "\"") && !strings.HasPrefix(l, "msgctxt") && !strings.HasPrefix(l, "msgid") && !strings.HasPrefix(l, "msgid_plural") && !strings.HasPrefix(l, "msgstr") { - return false + valid := []string{ + "\"", + "msgctxt", + "msgid", + "msgid_plural", + "msgstr", } - return true + for _, v := range valid { + if strings.HasPrefix(l, v) { + return true + } + } + + return false } // parseHeaders retrieves data from previously parsed headers func (po *Po) parseHeaders() { // Make sure we end with 2 carriage returns. - po.RawHeaders += "\n\n" + raw := po.Get("") + "\n\n" // Read - reader := bufio.NewReader(strings.NewReader(po.RawHeaders)) + reader := bufio.NewReader(strings.NewReader(raw)) tp := textproto.NewReader(reader) - mimeHeader, err := tp.ReadMIMEHeader() + var err error + + // Sync Headers write. + po.Lock() + defer po.Unlock() + + po.Headers, err = tp.ReadMIMEHeader() if err != nil { return } // Get/save needed headers - po.Language = mimeHeader.Get("Language") - po.PluralForms = mimeHeader.Get("Plural-Forms") + po.Language = po.Headers.Get("Language") + po.PluralForms = po.Headers.Get("Plural-Forms") // Parse Plural-Forms formula if po.PluralForms == "" { @@ -422,12 +427,12 @@ func (po *Po) Get(str string, vars ...interface{}) string { if po.translations != nil { if _, ok := po.translations[str]; ok { - return fmt.Sprintf(po.translations[str].get(), vars...) + return po.printf(po.translations[str].get(), vars...) } } // Return the same we received by default - return fmt.Sprintf(str, vars...) + return po.printf(str, vars...) } // GetN retrieves the (N)th plural form of translation for the given string. @@ -439,15 +444,14 @@ func (po *Po) GetN(str, plural string, n int, vars ...interface{}) string { if po.translations != nil { if _, ok := po.translations[str]; ok { - return fmt.Sprintf(po.translations[str].getN(po.pluralForm(n)), vars...) + return po.printf(po.translations[str].getN(po.pluralForm(n)), vars...) } } if n == 1 { - return fmt.Sprintf(str, vars...) + return po.printf(str, vars...) } - - return fmt.Sprintf(plural, vars...) + return po.printf(plural, vars...) } // GetC retrieves the corresponding translation for a given string in the given context. @@ -461,14 +465,14 @@ func (po *Po) GetC(str, ctx string, vars ...interface{}) string { if _, ok := po.contexts[ctx]; ok { if po.contexts[ctx] != nil { if _, ok := po.contexts[ctx][str]; ok { - return fmt.Sprintf(po.contexts[ctx][str].get(), vars...) + return po.printf(po.contexts[ctx][str].get(), vars...) } } } } // Return the string we received by default - return fmt.Sprintf(str, vars...) + return po.printf(str, vars...) } // GetNC retrieves the (N)th plural form of translation for the given string in the given context. @@ -482,14 +486,23 @@ func (po *Po) GetNC(str, plural string, n int, ctx string, vars ...interface{}) if _, ok := po.contexts[ctx]; ok { if po.contexts[ctx] != nil { if _, ok := po.contexts[ctx][str]; ok { - return fmt.Sprintf(po.contexts[ctx][str].getN(po.pluralForm(n)), vars...) + return po.printf(po.contexts[ctx][str].getN(po.pluralForm(n)), vars...) } } } } if n == 1 { + return po.printf(str, vars...) + } + return po.printf(plural, vars...) +} + +// printf applies text formatting only when needed to parse variables. +func (po *Po) printf(str string, vars ...interface{}) string { + if len(vars) > 0 { return fmt.Sprintf(str, vars...) } - return fmt.Sprintf(plural, vars...) + + return str } diff --git a/po_test.go b/po_test.go index 2291ac0..7eb9a7c 100644 --- a/po_test.go +++ b/po_test.go @@ -9,6 +9,9 @@ import ( func TestPo(t *testing.T) { // Set PO content str := ` +msgid "" +msgstr "" + # Initial comment # Headers below "Language: en\n" @@ -48,14 +51,6 @@ msgstr[0] "This one is the singular: %s" msgstr[1] "This one is the plural: %s" msgstr[2] "And this is the second plural form: %s" -msgid "This one has invalid syntax translations" -msgid_plural "Plural index" -msgstr[abc] "Wrong index" -msgstr[1 "Forgot to close brackets" -msgstr[0] "Badly formatted string' - -msgid "Invalid formatted id[] with no translations - msgctxt "Ctx" msgid "One with var: %s" msgid_plural "Several with vars: %s" @@ -75,11 +70,11 @@ msgstr "" msgid "Empty plural form singular" msgid_plural "Empty plural form" msgstr[0] "Singular translated" -msgstr[1] " +msgstr[1] "" msgid "More" msgstr "More translation" -" + ` // Write PO content to file @@ -152,17 +147,6 @@ msgstr "More translation" t.Errorf("Expected 'This are tests' but got '%s'", tr) } - // Test syntax error parsed translations - tr = po.Get("This one has invalid syntax translations") - if tr != "This one has invalid syntax translations" { - t.Errorf("Expected 'This one has invalid syntax translations' but got '%s'", tr) - } - - tr = po.GetN("This one has invalid syntax translations", "This are tests", 4) - if tr != "Plural index" { - t.Errorf("Expected 'Plural index' but got '%s'", tr) - } - // Test context translations v = "Test" tr = po.GetC("One with var: %s", "Ctx", v) @@ -214,9 +198,42 @@ msgstr "More translation" } } +func TestPlural(t *testing.T) { + // Set PO content + str := ` +msgid "" +msgstr "" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +msgid "Singular: %s" +msgid_plural "Plural: %s" +msgstr[0] "TR Singular: %s" +msgstr[1] "TR Plural: %s" +msgstr[2] "TR Plural 2: %s" + + +` + // Create po object + po := new(Po) + po.Parse(str) + + v := "Var" + tr := po.GetN("Singular: %s", "Plural: %s", 2, v) + if tr != "TR Plural: Var" { + t.Errorf("Expected 'TR Plural: Var' but got '%s'", tr) + } + + tr = po.GetN("Singular: %s", "Plural: %s", 1, v) + if tr != "TR Singular: Var" { + t.Errorf("Expected 'TR Singular: Var' but got '%s'", tr) + } +} + func TestPoHeaders(t *testing.T) { // Set PO content str := ` +msgid "" +msgstr "" # Initial comment # Headers below "Language: en\n" @@ -246,9 +263,30 @@ msgstr "Translated example" } } +func TestMissingPoHeadersSupport(t *testing.T) { + // Set PO content + str := ` +msgid "Example" +msgstr "Translated example" + ` + + // Create po object + po := new(Po) + + // Parse + po.Parse(str) + + // Check translation expected + if po.Get("Example") != "Translated example" { + t.Errorf("Expected 'Translated example' but got '%s'", po.Get("Example")) + } +} + func TestPluralFormsSingle(t *testing.T) { // Single form str := ` +msgid "" +msgstr "" "Plural-Forms: nplurals=1; plural=0;" # Some comment @@ -292,6 +330,8 @@ msgstr[3] "Plural form 3" func TestPluralForms2(t *testing.T) { // 2 forms str := ` +msgid "" +msgstr "" "Plural-Forms: nplurals=2; plural=n != 1;" # Some comment @@ -331,6 +371,8 @@ msgstr[3] "Plural form 3" func TestPluralForms3(t *testing.T) { // 3 forms str := ` +msgid "" +msgstr "" "Plural-Forms: nplurals=3; plural=n%10==1 && n%100!=11 ? 0 : n != 0 ? 1 : 2;" # Some comment @@ -378,6 +420,8 @@ msgstr[3] "Plural form 3" func TestPluralFormsSpecial(t *testing.T) { // 3 forms special str := ` +msgid "" +msgstr "" "Plural-Forms: nplurals=3;" "plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2;"