Create own plurals subpackage

fix Typo
This commit is contained in:
Josef Fröhle 2018-03-22 21:47:58 +01:00
parent 92b69ffa4c
commit a19c5fd581
22 changed files with 716 additions and 3806 deletions

8
Gopkg.lock generated
View file

@ -1,15 +1,9 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
branch = "master"
name = "github.com/mattn/kinako"
packages = ["ast","parser","vm"]
revision = "332c0a7e205a29536e672337a4bea6c7a96b04c1"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "d3069fabe2d6f79fe33ad88133e861db84aef0400f6b949c4e64395913b3ae97"
inputs-digest = "ab4fef131ee828e96ba67d31a7d690bd5f2f42040c6766b1b12fe856f87e0ff7"
solver-name = "gps-cdcl"
solver-version = 1

View file

@ -1,7 +1,6 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
@ -17,10 +16,15 @@
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
# name = "github.com/x/y"
# version = "2.4.0"
#
# [prune]
# non-go = false
# go-tests = true
# unused-packages = true
[[constraint]]
branch = "master"
name = "github.com/mattn/kinako"
[prune]
go-tests = true
unused-packages = true

431
plurals/compiler.go Normal file
View file

@ -0,0 +1,431 @@
/*
* Copyright (c) 2018 DeineAgentur UG https://www.deineagentur.com. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for full license information.
*/
package plurals
import (
"errors"
"fmt"
"regexp"
"strconv"
"strings"
)
type match struct {
openPos int
closePos int
}
var pat = regexp.MustCompile(`(\?|:|\|\||&&|==|!=|>=|>|<=|<|%|\d+|n)`)
type exprToken interface {
compile(tokens []string) (expr Expression, err error)
}
type testToken interface {
compile(tokens []string) (test test, err error)
}
type cmpTestBuilder func(val uint32, flipped bool) test
type logicTestBuild func(left test, right test) test
var ternaryToken ternaryStruct
type ternaryStruct struct{}
func (ternaryStruct) compile(tokens []string) (expr Expression, err error) {
main, err := splitTokens(tokens, "?")
if err != nil {
return expr, err
}
test, err := compileTest(strings.Join(main.Left, ""))
if err != nil {
return expr, err
}
actions, err := splitTokens(main.Right, ":")
if err != nil {
return expr, err
}
true_action, err := compileExpression(strings.Join(actions.Left, ""))
if err != nil {
return expr, err
}
false_action, err := compileExpression(strings.Join(actions.Right, ""))
if err != nil {
return expr, nil
}
return ternary{
test: test,
trueExpr: true_action,
falseExpr: false_action,
}, nil
}
var constToken constValStruct
type constValStruct struct{}
func (constValStruct) compile(tokens []string) (expr Expression, err error) {
if len(tokens) == 0 {
return expr, errors.New("got nothing instead of constant")
}
if len(tokens) != 1 {
return expr, fmt.Errorf("invalid constant: %s", strings.Join(tokens, ""))
}
i, err := strconv.Atoi(tokens[0])
if err != nil {
return expr, err
}
return constValue{value: i}, nil
}
func compileLogicTest(tokens []string, sep string, builder logicTestBuild) (test test, err error) {
split, err := splitTokens(tokens, sep)
if err != nil {
return test, err
}
left, err := compileTest(strings.Join(split.Left, ""))
if err != nil {
return test, err
}
right, err := compileTest(strings.Join(split.Right, ""))
if err != nil {
return test, err
}
return builder(left, right), nil
}
var orToken orStruct
type orStruct struct{}
func (orStruct) compile(tokens []string) (test test, err error) {
return compileLogicTest(tokens, "||", buildOr)
}
func buildOr(left test, right test) test {
return or{left: left, right: right}
}
var andToken andStruct
type andStruct struct{}
func (andStruct) compile(tokens []string) (test test, err error) {
return compileLogicTest(tokens, "&&", buildAnd)
}
func buildAnd(left test, right test) test {
return and{left: left, right: right}
}
func compileMod(tokens []string) (math math, err error) {
split, err := splitTokens(tokens, "%")
if err != nil {
return math, err
}
if len(split.Left) != 1 || split.Left[0] != "n" {
return math, errors.New("Modulus operation requires 'n' as left operand")
}
if len(split.Right) != 1 {
return math, errors.New("Modulus operation requires simple integer as right operand")
}
i, err := parseUint32(split.Right[0])
if err != nil {
return math, err
}
return mod{value: uint32(i)}, nil
}
func subPipe(modTokens []string, actionTokens []string, builder cmpTestBuilder, flipped bool) (test test, err error) {
modifier, err := compileMod(modTokens)
if err != nil {
return test, err
}
if len(actionTokens) != 1 {
return test, errors.New("can only get modulus of integer")
}
i, err := parseUint32(actionTokens[0])
if err != nil {
return test, err
}
action := builder(uint32(i), flipped)
return pipe{
modifier: modifier,
action: action,
}, nil
}
func compileEquality(tokens []string, sep string, builder cmpTestBuilder) (test test, err error) {
split, err := splitTokens(tokens, sep)
if err != nil {
return test, err
}
if len(split.Left) == 1 && split.Left[0] == "n" {
if len(split.Right) != 1 {
return test, errors.New("test can only compare n to integers")
}
i, err := parseUint32(split.Right[0])
if err != nil {
return test, err
}
return builder(i, false), nil
} else if len(split.Right) == 1 && split.Right[0] == "n" {
if len(split.Left) != 1 {
return test, errors.New("test can only compare n to integers")
}
i, err := parseUint32(split.Left[0])
if err != nil {
return test, err
}
return builder(i, true), nil
} else if contains(split.Left, "n") && contains(split.Left, "%") {
return subPipe(split.Left, split.Right, builder, false)
} else {
return test, errors.New("equality test must have 'n' as one of the two tests")
}
}
var eqToken eqStruct
type eqStruct struct{}
func (eqStruct) compile(tokens []string) (test test, err error) {
return compileEquality(tokens, "==", buildEq)
}
func buildEq(val uint32, flipped bool) test {
return equal{value: val}
}
var neqToken neqStruct
type neqStruct struct{}
func (neqStruct) compile(tokens []string) (test test, err error) {
return compileEquality(tokens, "!=", buildNeq)
}
func buildNeq(val uint32, flipped bool) test {
return notequal{value: val}
}
var gtToken gtStruct
type gtStruct struct{}
func (gtStruct) compile(tokens []string) (test test, err error) {
return compileEquality(tokens, ">", buildGt)
}
func buildGt(val uint32, flipped bool) test {
return gt{value: val, flipped: flipped}
}
var gteToken gteStruct
type gteStruct struct{}
func (gteStruct) compile(tokens []string) (test test, err error) {
return compileEquality(tokens, ">=", buildGte)
}
func buildGte(val uint32, flipped bool) test {
return gte{value: val, flipped: flipped}
}
var ltToken ltStruct
type ltStruct struct{}
func (ltStruct) compile(tokens []string) (test test, err error) {
return compileEquality(tokens, "<", buildLt)
}
func buildLt(val uint32, flipped bool) test {
return lt{value: val, flipped: flipped}
}
var lteToken lteStruct
type lteStruct struct{}
func (lteStruct) compile(tokens []string) (test test, err error) {
return compileEquality(tokens, "<=", buildLte)
}
func buildLte(val uint32, flipped bool) test {
return lte{value: val, flipped: flipped}
}
type testTokenDef struct {
op string
token testToken
}
var precedence = []testTokenDef{
{op: "||", token: orToken},
{op: "&&", token: andToken},
{op: "==", token: eqToken},
{op: "!=", token: neqToken},
{op: ">=", token: gteToken},
{op: ">", token: gtToken},
{op: "<=", token: lteToken},
{op: "<", token: ltToken},
}
type splitted struct {
Left []string
Right []string
}
// Find index of token in list of tokens
func index(tokens []string, sep string) int {
for index, token := range tokens {
if token == sep {
return index
}
}
return -1
}
// Split a list of tokens by a token into a splitted struct holding the tokens
// before and after the token to be split by.
func splitTokens(tokens []string, sep string) (s splitted, err error) {
index := index(tokens, sep)
if index == -1 {
return s, fmt.Errorf("'%s' not found in ['%s']", sep, strings.Join(tokens, "','"))
}
return splitted{
Left: tokens[:index],
Right: tokens[index+1:],
}, nil
}
// Scan a string for parenthesis
func scan(s string) <-chan match {
ch := make(chan match)
go func() {
depth := 0
opener := 0
for index, char := range s {
switch char {
case '(':
if depth == 0 {
opener = index
}
depth++
case ')':
depth--
if depth == 0 {
ch <- match{
openPos: opener,
closePos: index + 1,
}
}
}
}
close(ch)
}()
return ch
}
// Split the string into tokens
func split(s string) <- chan string {
ch := make(chan string)
go func() {
s = strings.Replace(s, " ", "", -1)
if !strings.Contains(s, "(") {
ch <- s
} else {
last := 0
end := len(s)
for info := range scan(s) {
if last != info.openPos {
ch <- s[last:info.openPos]
}
ch <- s[info.openPos:info.closePos]
last = info.closePos
}
if last != end {
ch <- s[last:]
}
}
close(ch)
}()
return ch
}
// Tokenizes a string into a list of strings, tokens grouped by parenthesis are
// not split! If the string starts with ( and ends in ), those are stripped.
func tokenize(s string) []string {
/*
TODO: Properly detect if the string starts with a ( and ends with a )
and that those two form a matching pair.
Eg: (foo) -> true; (foo)(bar) -> false;
*/
if s[0] == '(' && s[len(s)-1] == ')' {
s = s[1 : len(s)-1]
}
ret := []string{}
for chunk := range split(s) {
if len(chunk) != 0 {
if chunk[0] == '(' && chunk[len(chunk)-1] == ')' {
ret = append(ret, chunk)
} else {
for _, token := range pat.FindAllStringSubmatch(chunk, -1) {
ret = append(ret, token[0])
}
}
} else {
fmt.Printf("Empty chunk in string '%s'\n", s)
}
}
return ret
}
// Compile a string containing a plural form expression to a Expression object.
func Compile(s string) (expr Expression, err error) {
if s == "0" {
return constValue{value: 0}, nil
}
if !strings.Contains(s, "?") {
s += "?1:0"
}
return compileExpression(s)
}
// Check if a token is in a slice of strings
func contains(haystack []string, needle string) bool {
for _, s := range haystack {
if s == needle {
return true
}
}
return false
}
// Compiles an expression (ternary or constant)
func compileExpression(s string) (expr Expression, err error) {
tokens := tokenize(s)
if contains(tokens, "?") {
return ternaryToken.compile(tokens)
} else {
return constToken.compile(tokens)
}
}
// Compiles a test (comparison)
func compileTest(s string) (test test, err error) {
tokens := tokenize(s)
for _, tokenDef := range precedence {
if contains(tokens, tokenDef.op) {
return tokenDef.token.compile(tokens)
}
}
return test, errors.New("cannot compile")
}
func parseUint32(s string) (ui uint32, err error) {
i, err := strconv.ParseUint(s, 10, 32)
if err != nil {
return ui, err
} else {
return uint32(i), nil
}
}

50
plurals/compiler_test.go Normal file
View file

@ -0,0 +1,50 @@
/*
* Copyright (c) 2018 DeineAgentur UG https://www.deineagentur.com. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for full license information.
*/
package plurals
import (
"encoding/json"
"os"
"testing"
)
type fixture struct {
PluralForm string
Fixture []int
}
func TestCompiler(t *testing.T) {
f, err := os.Open("testdata/pluralforms.json")
if err != nil {
t.Fatal(err)
}
dec := json.NewDecoder(f)
var fixtures []fixture
err = dec.Decode(&fixtures)
if err != nil {
t.Fatal(err)
}
for _, data := range fixtures {
expr, err := Compile(data.PluralForm)
if err != nil {
t.Errorf("'%s' triggered error: %s", data.PluralForm, err)
} else if expr == nil {
t.Logf("'%s' compiled to nil", data.PluralForm)
t.Fail()
} else {
for n, e := range data.Fixture {
i := expr.Eval(uint32(n))
if i != e {
t.Logf("'%s' with n = %d, expected %d, got %d, compiled to %s", data.PluralForm, n, e, i, expr)
t.Fail()
}
if i == -1 {
break
}
}
}
}
}

44
plurals/expression.go Normal file
View file

@ -0,0 +1,44 @@
/*
* Copyright (c) 2018 DeineAgentur UG https://www.deineagentur.com. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for full license information.
*/
package plurals
// Expression is a plurals expression. Eval evaluates the expression for
// a given n value. Use plurals.Compile to generate Expression instances.
type Expression interface {
Eval(n uint32) int
}
type constValue struct {
value int
}
func (c constValue) Eval(n uint32) int {
return c.value
}
type test interface {
test(n uint32) bool
}
type ternary struct {
test test
trueExpr Expression
falseExpr Expression
}
func (t ternary) Eval(n uint32) int {
if t.test.test(n) {
if t.trueExpr == nil {
return -1
}
return t.trueExpr.Eval(n)
} else {
if t.falseExpr == nil {
return -1
}
return t.falseExpr.Eval(n)
}
}

18
plurals/math.go Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright (c) 2018 DeineAgentur UG https://www.deineagentur.com. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for full license information.
*/
package plurals
type math interface {
calc(n uint32) uint32
}
type mod struct {
value uint32
}
func (m mod) calc(n uint32) uint32 {
return n % m.value
}

1
plurals/testdata/pluralforms.json vendored Normal file

File diff suppressed because one or more lines are too long

109
plurals/tests.go Normal file
View file

@ -0,0 +1,109 @@
/*
* Copyright (c) 2018 DeineAgentur UG https://www.deineagentur.com. All rights reserved.
* Licensed under the MIT License. See LICENSE file in the project root for full license information.
*/
package plurals
type equal struct {
value uint32
}
func (e equal) test(n uint32) bool {
return n == e.value
}
type notequal struct {
value uint32
}
func (e notequal) test(n uint32) bool {
return n != e.value
}
type gt struct {
value uint32
flipped bool
}
func (e gt) test(n uint32) bool {
if e.flipped {
return e.value > n
} else {
return n > e.value
}
}
type lt struct {
value uint32
flipped bool
}
func (e lt) test(n uint32) bool {
if e.flipped {
return e.value < n
} else {
return n < e.value
}
}
type gte struct {
value uint32
flipped bool
}
func (e gte) test(n uint32) bool {
if e.flipped {
return e.value >= n
} else {
return n >= e.value
}
}
type lte struct {
value uint32
flipped bool
}
func (e lte) test(n uint32) bool {
if e.flipped {
return e.value <= n
} else {
return n <= e.value
}
}
type and struct {
left test
right test
}
func (e and) test(n uint32) bool {
if !e.left.test(n) {
return false
} else {
return e.right.test(n)
}
}
type or struct {
left test
right test
}
func (e or) test(n uint32) bool {
if e.left.test(n) {
return true
} else {
return e.right.test(n)
}
}
type pipe struct {
modifier math
action test
}
func (e pipe) test(n uint32) bool {
return e.action.test(e.modifier.calc(n))
}

51
po.go
View file

@ -9,7 +9,7 @@ import (
"strings"
"sync"
"github.com/mattn/kinako/vm"
"github.com/leonelquinteros/gotext/plurals"
)
type translation struct {
@ -33,7 +33,7 @@ func (t *translation) get() string {
}
}
// Return unstranlated id by default
// Return untranslated id by default
return t.id
}
@ -45,7 +45,7 @@ func (t *translation) getN(n int) string {
}
}
// Return unstranlated singular if corresponding
// Return untranslated singular if corresponding
if n == 0 {
return t.id
}
@ -89,8 +89,9 @@ type Po struct {
PluralForms string
// Parsed Plural-Forms header values
nplurals int
plural string
nplurals int
plural string
pluralforms plurals.Expression
// Storage
translations map[string]*translation
@ -107,7 +108,7 @@ type Po struct {
type parseState int
const (
head parseState = iota
head parseState = iota
msgCtxt
msgID
msgIDPlural
@ -377,6 +378,11 @@ func (po *Po) parseHeaders() {
case "plural":
po.plural = vs[1]
if expr, err := plurals.Compile(po.plural); err == nil {
po.pluralforms = expr
}
}
}
}
@ -387,35 +393,16 @@ func (po *Po) pluralForm(n int) int {
po.RLock()
defer po.RUnlock()
// Failsafe
if po.nplurals < 1 {
return 0
}
if po.plural == "" {
return 0
}
// Init compiler
env := vm.NewEnv()
env.Define("n", n)
plural, err := env.Execute(po.plural)
if err != nil {
return 0
}
if plural.Type().Name() == "bool" {
if plural.Bool() {
// Failure fallback
if po.pluralforms == nil {
/* Use the Germanic plural rule. */
if n == 1 {
return 0
} else {
return 1
}
// Else
return 0
}
if int(plural.Int()) > po.nplurals {
return 0
}
return int(plural.Int())
return po.pluralforms.Eval(uint32(n))
}
// Get retrieves the corresponding translation for the given string.

View file

@ -136,7 +136,7 @@ msgstr "More translation"
t.Errorf("Expected 'This one is the plural: Variable' but got '%s'", tr)
}
// Test inexistent translations
// Test not existent translations
tr = po.Get("This is a test")
if tr != "This is a test" {
t.Errorf("Expected 'This is a test' but got '%s'", tr)
@ -212,6 +212,37 @@ msgstr[1] "TR Plural: %s"
msgstr[2] "TR Plural 2: %s"
`
// Create po object
po := new(Po)
po.Parse(str)
v := "Var"
tr := po.GetN("Singular: %s", "Plural: %s", 2, v)
if tr != "TR Plural: Var" {
t.Errorf("Expected 'TR Plural: Var' but got '%s'", tr)
}
tr = po.GetN("Singular: %s", "Plural: %s", 1, v)
if tr != "TR Singular: Var" {
t.Errorf("Expected 'TR Singular: Var' but got '%s'", tr)
}
}
func TestPluralNoHeaderInformation(t *testing.T) {
// Set PO content
str := `
msgid ""
msgstr ""
msgid "Singular: %s"
msgid_plural "Plural: %s"
msgstr[0] "TR Singular: %s"
msgstr[1] "TR Plural: %s"
msgstr[2] "TR Plural 2: %s"
`
// Create po object
po := new(Po)

View file

@ -1,37 +0,0 @@
# kinako
Kinako is small VM written in Go.
![](https://raw.githubusercontent.com/mattn/kinako/master/kinako.png)
(Picture licensed under CC BY-SA 3.0 by wikipedia)
## Installation
Requires Go.
```
$ go get -u github.com/mattn/kinako
```
## Usage
Embedding the interpreter into your own program:
```Go
var env = vm.NewEnv()
env.Define("foo", 1)
val, err := env.Execute(`foo + 3`)
if err != nil {
panic(err)
}
fmt.Println(val)
```
# License
MIT
# Author
Yasuhiro Matsumoto (a.k.a mattn)

View file

@ -1,18 +0,0 @@
package main
import (
"fmt"
"log"
"github.com/mattn/kinako/vm"
)
func main() {
env := vm.NewEnv()
v, err := env.Execute(`foo=1; foo+3`)
if err != nil {
log.Fatal(err)
}
fmt.Println(v)
}

View file

@ -1,112 +0,0 @@
package ast
type Token struct {
Tok int
Lit string
}
// Position provides interface to store code locations.
type Position struct {
Line int
Column int
}
// Expr provides all of interfaces for expression.
type Expr interface {
expr()
}
// ExprImpl provide commonly implementations for Expr.
type ExprImpl struct {
}
// expr provide restraint interface.
func (x *ExprImpl) expr() {}
// NumberExpr provide Number expression.
type NumberExpr struct {
ExprImpl
Lit string
}
// UnaryExpr provide unary minus expression. ex: -1, ^1, ~1.
type UnaryExpr struct {
ExprImpl
Operator string
Expr Expr
}
// IdentExpr provide identity expression.
type IdentExpr struct {
ExprImpl
Lit string
}
// Stmt provides all of interfaces for statement.
type Stmt interface {
stmt()
}
// StmtImpl provide commonly implementations for Stmt..
type StmtImpl struct {
}
// stmt provide restraint interface.
func (x *StmtImpl) stmt() {}
// LetsStmt provide multiple statement of let.
type LetsStmt struct {
StmtImpl
Lhss []Expr
Operator string
Rhss []Expr
}
// StringExpr provide String expression.
type StringExpr struct {
ExprImpl
Lit string
}
type TernaryOpExpr struct {
ExprImpl
Expr Expr
Lhs Expr
Rhs Expr
}
// CallExpr provide calling expression.
type CallExpr struct {
ExprImpl
Func interface{}
Name string
SubExprs []Expr
}
// ParenExpr provide parent block expression.
type ParenExpr struct {
ExprImpl
SubExpr Expr
}
// BinOpExpr provide binary operator expression.
type BinOpExpr struct {
ExprImpl
Lhs Expr
Operator string
Rhs Expr
}
// ExprStmt provide expression statement.
type ExprStmt struct {
StmtImpl
Expr Expr
}
// LetStmt provide statement of let.
type LetStmt struct {
StmtImpl
Lhs Expr
Operator string
Rhs Expr
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 450 KiB

View file

@ -1,4 +0,0 @@
all : parser.go
parser.go : parser.go.y
goyacc -o $@ parser.go.y

View file

@ -1,427 +0,0 @@
package parser
import (
"errors"
"fmt"
"unicode"
"github.com/mattn/kinako/ast"
)
const (
EOF = -1 // End of file.
EOL = '\n' // End of line.
)
// Error provides a convenient interface for handling runtime error.
// It can be Error inteface with type cast which can call Pos().
type Error struct {
Message string
Filename string
Fatal bool
}
// Error returns the error message.
func (e *Error) Error() string {
return e.Message
}
// Scanner stores informations for lexer.
type Scanner struct {
src []rune
offset int
lineHead int
line int
}
// Init resets code to scan.
func (s *Scanner) Init(src string) {
s.src = []rune(src)
}
// Scan analyses token, and decide identify or literals.
func (s *Scanner) Scan() (tok int, lit string, pos ast.Position, err error) {
retry:
s.skipBlank()
pos = s.pos()
switch ch := s.peek(); {
case isLetter(ch):
tok = IDENT
lit, err = s.scanIdentifier()
if err != nil {
return
}
case isDigit(ch):
tok = NUMBER
lit, err = s.scanNumber()
if err != nil {
return
}
case ch == '"':
tok = STRING
lit, err = s.scanString('"')
if err != nil {
return
}
case ch == '\'':
tok = STRING
lit, err = s.scanString('\'')
if err != nil {
return
}
case ch == '`':
tok = STRING
lit, err = s.scanRawString()
if err != nil {
return
}
default:
switch ch {
case EOF:
tok = EOF
case '#':
for !isEOL(s.peek()) {
s.next()
}
goto retry
case '!':
s.next()
switch s.peek() {
case '=':
tok = NEQ
lit = "!="
default:
s.back()
tok = int(ch)
lit = string(ch)
}
case '=':
s.next()
switch s.peek() {
case '=':
tok = EQEQ
lit = "=="
default:
s.back()
tok = int(ch)
lit = string(ch)
}
case '+':
tok = int(ch)
lit = string(ch)
case '-':
tok = int(ch)
lit = string(ch)
case '*':
tok = int(ch)
lit = string(ch)
case '/':
tok = int(ch)
lit = string(ch)
case '>':
s.next()
switch s.peek() {
case '=':
tok = GE
lit = ">="
case '>':
tok = SHIFTRIGHT
lit = ">>"
default:
s.back()
tok = int(ch)
lit = string(ch)
}
case '<':
s.next()
switch s.peek() {
case '=':
tok = LE
lit = "<="
case '<':
tok = SHIFTLEFT
lit = "<<"
default:
s.back()
tok = int(ch)
lit = string(ch)
}
case '|':
s.next()
switch s.peek() {
case '|':
tok = OROR
lit = "||"
default:
s.back()
tok = int(ch)
lit = string(ch)
}
case '&':
s.next()
switch s.peek() {
case '&':
tok = ANDAND
lit = "&&"
default:
s.back()
tok = int(ch)
lit = string(ch)
}
case '.':
tok = int(ch)
lit = string(ch)
case '\n':
tok = int(ch)
lit = string(ch)
case '(', ')', ':', ';', '%', '?', '{', '}', ',', '[', ']', '^':
tok = int(ch)
lit = string(ch)
default:
err = fmt.Errorf(`syntax error "%s"`, string(ch))
tok = int(ch)
lit = string(ch)
return
}
s.next()
}
return
}
// isLetter returns true if the rune is a letter for identity.
func isLetter(ch rune) bool {
return unicode.IsLetter(ch) || ch == '_'
}
// isDigit returns true if the rune is a number.
func isDigit(ch rune) bool {
return '0' <= ch && ch <= '9'
}
// isHex returns true if the rune is a hex digits.
func isHex(ch rune) bool {
return ('0' <= ch && ch <= '9') || ('a' <= ch && ch <= 'f') || ('A' <= ch && ch <= 'F')
}
// isEOL returns true if the rune is at end-of-line or end-of-file.
func isEOL(ch rune) bool {
return ch == '\n' || ch == -1
}
// isBlank returns true if the rune is empty character..
func isBlank(ch rune) bool {
return ch == ' ' || ch == '\t' || ch == '\r'
}
// peek returns current rune in the code.
func (s *Scanner) peek() rune {
if s.reachEOF() {
return EOF
}
return s.src[s.offset]
}
// next moves offset to next.
func (s *Scanner) next() {
if !s.reachEOF() {
if s.peek() == '\n' {
s.lineHead = s.offset + 1
s.line++
}
s.offset++
}
}
// current returns the current offset.
func (s *Scanner) current() int {
return s.offset
}
// offset sets the offset value.
func (s *Scanner) set(o int) {
s.offset = o
}
// back moves back offset once to top.
func (s *Scanner) back() {
s.offset--
}
// reachEOF returns true if offset is at end-of-file.
func (s *Scanner) reachEOF() bool {
return len(s.src) <= s.offset
}
// pos returns the position of current.
func (s *Scanner) pos() ast.Position {
return ast.Position{Line: s.line + 1, Column: s.offset - s.lineHead + 1}
}
// skipBlank moves position into non-black character.
func (s *Scanner) skipBlank() {
for isBlank(s.peek()) {
s.next()
}
}
// scanIdentifier returns identifier begining at current position.
func (s *Scanner) scanIdentifier() (string, error) {
var ret []rune
for {
if !isLetter(s.peek()) && !isDigit(s.peek()) {
break
}
ret = append(ret, s.peek())
s.next()
}
return string(ret), nil
}
// scanNumber returns number begining at current position.
func (s *Scanner) scanNumber() (string, error) {
var ret []rune
ch := s.peek()
ret = append(ret, ch)
s.next()
if ch == '0' && s.peek() == 'x' {
ret = append(ret, s.peek())
s.next()
for isHex(s.peek()) {
ret = append(ret, s.peek())
s.next()
}
} else {
for isDigit(s.peek()) || s.peek() == '.' {
ret = append(ret, s.peek())
s.next()
}
if s.peek() == 'e' {
ret = append(ret, s.peek())
s.next()
if isDigit(s.peek()) || s.peek() == '+' || s.peek() == '-' {
ret = append(ret, s.peek())
s.next()
for isDigit(s.peek()) || s.peek() == '.' {
ret = append(ret, s.peek())
s.next()
}
}
for isDigit(s.peek()) || s.peek() == '.' {
ret = append(ret, s.peek())
s.next()
}
}
if isLetter(s.peek()) {
return "", errors.New("identifier starts immediately after numeric literal")
}
}
return string(ret), nil
}
// scanRawString returns raw-string starting at current position.
func (s *Scanner) scanRawString() (string, error) {
var ret []rune
for {
s.next()
if s.peek() == EOF {
return "", errors.New("unexpected EOF")
break
}
if s.peek() == '`' {
s.next()
break
}
ret = append(ret, s.peek())
}
return string(ret), nil
}
// scanString returns string starting at current position.
// This handles backslash escaping.
func (s *Scanner) scanString(l rune) (string, error) {
var ret []rune
eos:
for {
s.next()
switch s.peek() {
case EOL:
return "", errors.New("unexpected EOL")
case EOF:
return "", errors.New("unexpected EOF")
case l:
s.next()
break eos
case '\\':
s.next()
switch s.peek() {
case 'b':
ret = append(ret, '\b')
continue
case 'f':
ret = append(ret, '\f')
continue
case 'r':
ret = append(ret, '\r')
continue
case 'n':
ret = append(ret, '\n')
continue
case 't':
ret = append(ret, '\t')
continue
}
ret = append(ret, s.peek())
continue
default:
ret = append(ret, s.peek())
}
}
return string(ret), nil
}
// Lexer provides inteface to parse codes.
type Lexer struct {
s *Scanner
lit string
pos ast.Position
e error
stmts []ast.Stmt
}
// Lex scans the token and literals.
func (l *Lexer) Lex(lval *yySymType) int {
tok, lit, pos, err := l.s.Scan()
if err != nil {
l.e = &Error{Message: fmt.Sprintf("%s", err.Error()), Fatal: true}
}
lval.tok = ast.Token{Tok: tok, Lit: lit}
l.lit = lit
l.pos = pos
return tok
}
// Error sets parse error.
func (l *Lexer) Error(msg string) {
l.e = &Error{Message: msg, Fatal: false}
}
// Parser provides way to parse the code using Scanner.
func Parse(s *Scanner) ([]ast.Stmt, error) {
l := Lexer{s: s}
if yyParse(&l) != 0 {
return nil, l.e
}
return l.stmts, l.e
}
func EnableErrorVerbose() {
yyErrorVerbose = true
}
// ParserSrc provides way to parse the code from source.
func ParseSrc(src string) ([]ast.Stmt, error) {
scanner := &Scanner{
src: []rune(src),
}
return Parse(scanner)
}

View file

@ -1,778 +0,0 @@
//line parser.go.y:2
package parser
import __yyfmt__ "fmt"
//line parser.go.y:2
import (
"github.com/mattn/kinako/ast"
)
//line parser.go.y:16
type yySymType struct {
yys int
compstmt []ast.Stmt
stmts []ast.Stmt
stmt ast.Stmt
expr ast.Expr
exprs []ast.Expr
tok ast.Token
term ast.Token
terms ast.Token
opt_terms ast.Token
}
const IDENT = 57346
const NUMBER = 57347
const STRING = 57348
const EQEQ = 57349
const NEQ = 57350
const GE = 57351
const LE = 57352
const OROR = 57353
const ANDAND = 57354
const POW = 57355
const SHIFTLEFT = 57356
const SHIFTRIGHT = 57357
const PLUSPLUS = 57358
const MINUSMINUS = 57359
const UNARY = 57360