From fd42cc208a6d0e4ffc62a5e44d4a1caaf48c25af Mon Sep 17 00:00:00 2001
From: zhouzhihong <zhouzhihong@xintech.co>
Date: Wed, 24 Aug 2022 23:25:24 +0800
Subject: [PATCH 1/3] Try to fixed postion of token for unicode.

---
 dfa/dfa_helpers_test.go                  |  42 +-
 dfa/gen_test.go                          |   4 +-
 doc.go                                   | 116 ++++
 examples/sensors-parser/.gitignore       |   1 +
 examples/sensors-parser/README.md        | 119 ++++
 examples/sensors-parser/ast.go           |  53 ++
 examples/sensors-parser/main.go          |  67 +++
 examples/sensors-parser/sensors.conf     |   4 +
 examples/sensors-parser/sensors.y        | 101 ++++
 examples/sensors-parser/sensors_golex.go | 115 ++++
 examples/sensors-parser/y.go             | 665 +++++++++++++++++++++++
 examples/sensors/main.go                 | 130 +++++
 frontend/ast.go                          |  12 +-
 frontend/desugar.go                      |   2 +-
 frontend/desugar_test.go                 |  11 +-
 frontend/frontend_test.go                |  52 +-
 frontend/parser.go                       |  26 +-
 grammar                                  |  36 ++
 lexc/main.go                             |   2 +-
 lexer.go                                 |  17 +-
 lexer_test.go                            |  98 ++--
 machines/dfa_machine.go                  |   5 +-
 machines/machine.go                      |  28 +-
 machines/machine_test.go                 |  22 +-
 24 files changed, 1568 insertions(+), 160 deletions(-)
 create mode 100644 doc.go
 create mode 100644 examples/sensors-parser/.gitignore
 create mode 100644 examples/sensors-parser/README.md
 create mode 100644 examples/sensors-parser/ast.go
 create mode 100644 examples/sensors-parser/main.go
 create mode 100644 examples/sensors-parser/sensors.conf
 create mode 100644 examples/sensors-parser/sensors.y
 create mode 100644 examples/sensors-parser/sensors_golex.go
 create mode 100644 examples/sensors-parser/y.go
 create mode 100644 examples/sensors/main.go
 create mode 100644 grammar

diff --git a/dfa/dfa_helpers_test.go b/dfa/dfa_helpers_test.go
index ac6a8e7..ea17d75 100644
--- a/dfa/dfa_helpers_test.go
+++ b/dfa/dfa_helpers_test.go
@@ -34,7 +34,7 @@ func TestLabeledAst(x *testing.T) {
 	for _, regex := range []string{
 		"a", "b", "asdf", "s|a", "sdf*", "(sdf)+(asdf)*", "w|(s|e)*(s)+(s?fe)**", "(a|we|f*|s*?)|W(LSD)Adf[23-s]",
 	} {
-		ast, err := frontend.Parse([]rune(regex))
+		ast, err := frontend.Parse([]byte(regex))
 		t.AssertNil(err)
 		verify(frontend.DesugarRanges(ast))
 	}
@@ -59,7 +59,7 @@ func followEquals(follow []map[int]bool, expected [][]int) bool {
 
 func testFollow(x *testing.T, regex string, expectedPos []frontend.AST, expectedFollows [][]int) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune(regex))
+	ast, err := frontend.Parse([]byte(regex))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	positions := lAst.Positions
@@ -334,7 +334,7 @@ func TestFollowNested(x *testing.T) {
 
 func TestMatchesEmptyString_char(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a"))
+	ast, err := frontend.Parse([]byte("a"))
 	t.AssertNil(err)
 	nullable := Label(ast).MatchesEmptyString()
 	t.Assert(!nullable[len(nullable)-1], "character should not match the empty string, %v", ast)
@@ -342,7 +342,7 @@ func TestMatchesEmptyString_char(x *testing.T) {
 
 func TestMatchesEmptyString_range(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("[a-z]"))
+	ast, err := frontend.Parse([]byte("[a-z]"))
 	t.AssertNil(err)
 	nullable := Label(ast).MatchesEmptyString()
 	t.Assert(!nullable[len(nullable)-1], "range should not match the empty string, %v", ast)
@@ -350,7 +350,7 @@ func TestMatchesEmptyString_range(x *testing.T) {
 
 func TestMatchesEmptyString_maybe(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a?"))
+	ast, err := frontend.Parse([]byte("a?"))
 	t.AssertNil(err)
 	nullable := Label(ast).MatchesEmptyString()
 	t.Assert(nullable[len(nullable)-1], "maybe should match the empty string, %v", ast)
@@ -358,7 +358,7 @@ func TestMatchesEmptyString_maybe(x *testing.T) {
 
 func TestMatchesEmptyString_star(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a*"))
+	ast, err := frontend.Parse([]byte("a*"))
 	t.AssertNil(err)
 	nullable := Label(ast).MatchesEmptyString()
 	t.Assert(nullable[len(nullable)-1], "star should match the empty string, %v", ast)
@@ -366,11 +366,11 @@ func TestMatchesEmptyString_star(x *testing.T) {
 
 func TestMatchesEmptyString_plus(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a+"))
+	ast, err := frontend.Parse([]byte("a+"))
 	t.AssertNil(err)
 	nullable := Label(ast).MatchesEmptyString()
 	t.Assert(!nullable[len(nullable)-1], "a+ should not match the empty string, %v", ast)
-	ast, err = frontend.Parse([]rune("a?+"))
+	ast, err = frontend.Parse([]byte("a?+"))
 	t.AssertNil(err)
 	nullable = Label(ast).MatchesEmptyString()
 	t.Assert(nullable[len(nullable)-1], "a?+ should match the empty string, %v", ast)
@@ -393,7 +393,7 @@ func TestMatchesEmptyString_concat(x *testing.T) {
 }
 
 func testMatchesEmptyString(t *test.T, regex string, matches bool, message string, args ...interface{}) {
-	ast, err := frontend.Parse([]rune(regex))
+	ast, err := frontend.Parse([]byte(regex))
 	t.AssertNil(err)
 	nullable := Label(ast).MatchesEmptyString()
 	t.Assert(nullable[len(nullable)-1] == matches, message, args...)
@@ -429,7 +429,7 @@ func astList(lAst *LabeledAST, l []int) []frontend.AST {
 
 func TestFirst_char(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a"))
+	ast, err := frontend.Parse([]byte("a"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	first := []frontend.AST{
@@ -440,7 +440,7 @@ func TestFirst_char(x *testing.T) {
 
 func TestLast_char(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a"))
+	ast, err := frontend.Parse([]byte("a"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	last := []frontend.AST{
@@ -452,7 +452,7 @@ func TestLast_char(x *testing.T) {
 
 func TestFirst_range(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("[a-z]"))
+	ast, err := frontend.Parse([]byte("[a-z]"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	first := []frontend.AST{
@@ -463,7 +463,7 @@ func TestFirst_range(x *testing.T) {
 
 func TestLast_range(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("[a-z]"))
+	ast, err := frontend.Parse([]byte("[a-z]"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	last := []frontend.AST{
@@ -475,7 +475,7 @@ func TestLast_range(x *testing.T) {
 
 func TestFirst_ops(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a?*+"))
+	ast, err := frontend.Parse([]byte("a?*+"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	first := []frontend.AST{
@@ -487,7 +487,7 @@ func TestFirst_ops(x *testing.T) {
 
 func TestLast_ops(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a*?+"))
+	ast, err := frontend.Parse([]byte("a*?+"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	last := []frontend.AST{
@@ -499,7 +499,7 @@ func TestLast_ops(x *testing.T) {
 
 func TestFirst_alt(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a|b"))
+	ast, err := frontend.Parse([]byte("a|b"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	first := []frontend.AST{
@@ -511,7 +511,7 @@ func TestFirst_alt(x *testing.T) {
 
 func TestLast_alt(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a|b"))
+	ast, err := frontend.Parse([]byte("a|b"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	last := []frontend.AST{
@@ -524,7 +524,7 @@ func TestLast_alt(x *testing.T) {
 
 func TestFirst_concat(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("a?b?c?de"))
+	ast, err := frontend.Parse([]byte("a?b?c?de"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	first := []frontend.AST{
@@ -538,7 +538,7 @@ func TestFirst_concat(x *testing.T) {
 
 func TestLast_concat(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("abc?d?e?"))
+	ast, err := frontend.Parse([]byte("abc?d?e?"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	last := []frontend.AST{
@@ -553,7 +553,7 @@ func TestLast_concat(x *testing.T) {
 
 func TestLast_concat2(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("abc*d*e*"))
+	ast, err := frontend.Parse([]byte("abc*d*e*"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	last := []frontend.AST{
@@ -568,7 +568,7 @@ func TestLast_concat2(x *testing.T) {
 
 func TestLast_concat3(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := frontend.Parse([]rune("abc+d+e+"))
+	ast, err := frontend.Parse([]byte("abc+d+e+"))
 	t.AssertNil(err)
 	lAst := Label(ast)
 	last := []frontend.AST{
diff --git a/dfa/gen_test.go b/dfa/gen_test.go
index 1a085ea..30b39e2 100644
--- a/dfa/gen_test.go
+++ b/dfa/gen_test.go
@@ -8,7 +8,7 @@ import (
 )
 
 func mustParse(regex string) frontend.AST {
-	ast, err := frontend.Parse([]rune(regex))
+	ast, err := frontend.Parse([]byte(regex))
 	if err != nil {
 		panic(err)
 	}
@@ -16,7 +16,7 @@ func mustParse(regex string) frontend.AST {
 }
 
 func testGen(t *test.T, regex, text string, matchID int) {
-	ast, err := frontend.Parse([]rune(regex))
+	ast, err := frontend.Parse([]byte(regex))
 	t.AssertNil(err)
 	testGenMatch(t, ast, text, matchID)
 }
diff --git a/doc.go b/doc.go
new file mode 100644
index 0000000..efe744d
--- /dev/null
+++ b/doc.go
@@ -0,0 +1,116 @@
+// Package lexmachine is a full lexical analysis framework for the Go
+// programming language. It supports a restricted but usable set of regular
+// expressions appropriate for writing lexers for complex programming
+// languages. The framework also supports sub-lexers and non-regular lexing
+// through an "escape hatch" which allows the users to consume any number of
+// further bytes after a match. So if you want to support nested C-style
+// comments or other paired structures you can do so at the lexical analysis
+// stage.
+//
+// For a tutorial see
+// http://hackthology.com/writing-a-lexer-in-go-with-lexmachine.html
+//
+// Example of defining a lexer
+//
+//     // CreateLexer defines a lexer for the graphviz dot language.
+//     func CreateLexer() (*lexmachine.Lexer, error) {
+//         lexer := lexmachine.NewLexer()
+//
+//         for _, lit := range Literals {
+//             r := "\\" + strings.Join(strings.Split(lit, ""), "\\")
+//             lexer.Add([]byte(r), token(lit))
+//         }
+//         for _, name := range Keywords {
+//             lexer.Add([]byte(strings.ToLower(name)), token(name))
+//         }
+//
+//         lexer.Add([]byte(`//[^\n]*\n?`), token("COMMENT"))
+//         lexer.Add([]byte(`/\*([^*]|\r|\n|(\*+([^*/]|\r|\n)))*\*+/`), token("COMMENT"))
+//         lexer.Add([]byte(`([a-z]|[A-Z])([a-z]|[A-Z]|[0-9]|_)*`), token("ID"))
+//         lexer.Add([]byte(`"([^\\"]|(\\.))*"`), token("ID"))
+//         lexer.Add([]byte("( |\t|\n|\r)+"), skip)
+//         lexer.Add([]byte(`\<`),
+//             func(scan *lexmachine.Scanner, match *machines.Match) (interface{}, error) {
+//                 str := make([]byte, 0, 10)
+//                 str = append(str, match.Bytes...)
+//                 brackets := 1
+//                 match.EndLine = match.StartLine
+//                 match.EndColumn = match.StartColumn
+//                 for tc := scan.TC; tc < len(scan.Text); tc++ {
+//                     str = append(str, scan.Text[tc])
+//                     match.EndColumn += 1
+//                     if scan.Text[tc] == '\n' {
+//                         match.EndLine += 1
+//                     }
+//                     if scan.Text[tc] == '<' {
+//                         brackets += 1
+//                     } else if scan.Text[tc] == '>' {
+//                         brackets -= 1
+//                     }
+//                     if brackets == 0 {
+//                         match.TC = scan.TC
+//                         scan.TC = tc + 1
+//                         match.Bytes = str
+//                         return token("ID")(scan, match)
+//                     }
+//                 }
+//                 return nil,
+//                     fmt.Errorf("unclosed HTML literal starting at %d, (%d, %d)",
+//                         match.TC, match.StartLine, match.StartColumn)
+//             },
+//         )
+//
+//         err := lexer.Compile()
+//         if err != nil {
+//             return nil, err
+//         }
+//         return lexer, nil
+//     }
+//
+//     func token(name string) lex.Action {
+//         return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+//             return s.Token(TokenIds[name], string(m.Bytes), m), nil
+//         }
+//     }
+//
+// Example of using a lexer
+//
+//     func ExampleLex() error {
+//         lexer, err := CreateLexer()
+//         if err != nil {
+//             return err
+//         }
+//         scanner, err := lexer.Scanner([]byte(`digraph {
+//           rankdir=LR;
+//           a [label="a" shape=box];
+//           c [<label>=<<u>C</u>>];
+//           b [label="bb"];
+//           a -> c;
+//           c -> b;
+//           d -> c;
+//           b -> a;
+//           b -> e;
+//           e -> f;
+//         }`))
+//         if err != nil {
+//             return err
+//         }
+//         fmt.Println("Type    | Lexeme     | Position")
+//         fmt.Println("--------+------------+------------")
+//         for tok, err, eos := scanner.Next(); !eos; tok, err, eos = scanner.Next() {
+//             if err != nil {
+//                 return err
+//             }
+//             token := tok.(*lexmachine.Token)
+//             fmt.Printf("%-7v | %-10v | %v:%v-%v:%v\n",
+//                 dot.Tokens[token.Type],
+//                 string(token.Lexeme),
+//                 token.StartLine,
+//                 token.StartColumn,
+//                 token.EndLine,
+//                 token.EndColumn)
+//         }
+//         return nil
+//     }
+//
+package lexmachine
diff --git a/examples/sensors-parser/.gitignore b/examples/sensors-parser/.gitignore
new file mode 100644
index 0000000..2652053
--- /dev/null
+++ b/examples/sensors-parser/.gitignore
@@ -0,0 +1 @@
+y.output
diff --git a/examples/sensors-parser/README.md b/examples/sensors-parser/README.md
new file mode 100644
index 0000000..9a76b68
--- /dev/null
+++ b/examples/sensors-parser/README.md
@@ -0,0 +1,119 @@
+# Parse a sensors.conf file with goyacc and lexmachine
+
+This is an example for how to integrate lexmachine with the [standard
+yacc](http://godoc.org/golang.org/x/tools/cmd/goyacc) implementation for Go.
+Yacc is its own weird and interesting language for specifying bottom up shift
+reduce parsers. You can "easily" use lexmachine with yacc but it does require
+some understanding of
+
+1.  How yacc works (eg. the things it generates)
+2.  How to use those generated definitions in your code
+
+## Running the example
+
+```sh
+$ go generate -x -v gitea.xintech.co/zhouzhihong/lexmachine/examples/sensors-parser
+$ go install gitea.xintech.co/zhouzhihong/lexmachine/examples/sensors-parser
+$ cat examples/sensors-parser/sensors.conf | sensors-parser
+```
+
+## Partial Explanation
+
+Yacc controls the definitions for Tokens with its `%token` directives (see the
+`sensors.y` file. You will use those definitions in your lexer. An example of
+how to do this is in `sensors_golex.go`.
+
+Second, Yacc expects the lexer to conform to the following interface:
+
+```go
+type yyLexer interface {
+   // Lex gets the next token and puts it in lval
+   Lex(lval *yySymType) (tokenType int)
+   // Error is called on parse error (it should probably panic?)
+   Error(message string)
+ }
+```
+
+The `yySymType` is generate from Yacc via the `%union` directive. The tokenType
+is the token identifier. However, the tokenType needs to be in the correct range
+for Yacc which starts at `yyPrivate`. The way to get the types identified
+correctly is to set it as `return token.Type + yyPrivate - 1`. See
+`sensors_golex.go` for a full example.
+
+Yacc in its own special way has each production "return" a yySymType which
+serves as *both* an AST node *and* a token. Thus, my definition for yySymType
+is:
+
+```yacc
+%union{
+    token *lexmachine.Token
+    ast   *Node
+}
+```
+
+This lets you construct an AST while parsing:
+
+```yacc
+Unary : DASH Factor             { $$.ast = NewNode("negate", $1.token).AddKid($2.ast) }
+      | BACKTICK Factor         { $$.ast = NewNode("`", $1.token).AddKid($2.ast) }
+      | CARROT Factor           { $$.ast = NewNode("^", $1.token).AddKid($2.ast) }
+      | Factor                  { $$.ast = $1.ast }
+      ;
+
+Factor : NAME                   { $$.ast = NewNode("name", $1.token) }
+       | NUMBER                 { $$.ast = NewNode("number", $1.token) }
+       | AT                     { $$.ast = NewNode("@", $1.token) }
+       | LPAREN Expr RPAREN     { $$.ast = $2.ast }
+       ;
+```
+
+Finally, yacc does not provide any means of returning anything from the parser.
+To deal with this the lexer you provide needs to have a field which provides the
+result back to the caller:
+
+```go
+type golex struct {
+    *lexmachine.Scanner
+    stmts []*Node
+}
+```
+
+In the example `stmts` provides the parsed statements from the file back to the
+caller of the parser:
+
+```go
+func parse(lexer *lexmachine.Lexer, fin io.Reader) (stmts []*Node, err error) {
+    defer func() {
+        if e := recover(); e != nil {
+            switch e.(type) {
+            case error:
+                err = e.(error)
+                stmts = nil
+            default:
+                panic(e)
+            }
+        }
+    }()
+    text, err := ioutil.ReadAll(fin)
+    if err != nil {
+        return nil, err
+    }
+    scanner, err := newGoLex(lexer, text)
+    if err != nil {
+        return nil, err
+    }
+    yyParse(scanner)
+    return scanner.stmts, nil
+}
+```
+
+Since `yyLexer` is an interface in goyacc their is some casting involved to
+populate `stmts` (note `yylex` is a magic variable in yacc that refers to the
+lexer object you provided):
+
+```yacc
+Line : Stmt NEWLINE             { yylex.(*golex).stmts = append(yylex.(*golex).stmts, $1.ast) }
+     | NEWLINE
+     ;
+
+```
diff --git a/examples/sensors-parser/ast.go b/examples/sensors-parser/ast.go
new file mode 100644
index 0000000..469cae5
--- /dev/null
+++ b/examples/sensors-parser/ast.go
@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"gitea.xintech.co/zhouzhihong/lexmachine"
+)
+
+// Node is a very simple AST/Parse Tree node. It stores a Name (required), a
+// Token (optional), and any child nodes.
+type Node struct {
+	Name     string
+	Token    *lexmachine.Token
+	Children []*Node
+}
+
+// NewNode makes a node from a name and a token. The token may be nil.
+func NewNode(name string, token *lexmachine.Token) *Node {
+	return &Node{
+		Name:  name,
+		Token: token,
+	}
+}
+
+// AddKid puts a node at the end of the child list
+func (n *Node) AddKid(kid *Node) *Node {
+	n.Children = append(n.Children, kid)
+	return n
+}
+
+// PrependKid puts a node at the beginning of the child list
+func (n *Node) PrependKid(kid *Node) *Node {
+	kids := append(make([]*Node, 0, cap(n.Children)+1), kid)
+	n.Children = append(kids, n.Children...)
+	return n
+}
+
+// String humanizes the tree starting at the current node.
+func (n *Node) String() string {
+	parts := make([]string, 0, len(n.Children))
+	parts = append(parts, n.Name)
+	if n.Token != nil && string(n.Token.Lexeme) != n.Name {
+		parts = append(parts, fmt.Sprintf("%q", string(n.Token.Lexeme)))
+	}
+	for _, k := range n.Children {
+		parts = append(parts, k.String())
+	}
+	if len(parts) > 1 {
+		return fmt.Sprintf("(%v)", strings.Join(parts, " "))
+	}
+	return strings.Join(parts, " ")
+}
diff --git a/examples/sensors-parser/main.go b/examples/sensors-parser/main.go
new file mode 100644
index 0000000..f35281c
--- /dev/null
+++ b/examples/sensors-parser/main.go
@@ -0,0 +1,67 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"gitea.xintech.co/zhouzhihong/lexmachine"
+	"github.com/timtadh/getopt"
+)
+
+func main() {
+	short := "h"
+	long := []string{
+		"help",
+	}
+
+	_, optargs, err := getopt.GetOpt(os.Args[1:], short, long)
+	if err != nil {
+		log.Print(err)
+		log.Println("try --help")
+		os.Exit(1)
+	}
+
+	for _, oa := range optargs {
+		switch oa.Opt() {
+		case "-h", "--help":
+			fmt.Println("parse a sensors.conf")
+			os.Exit(0)
+		}
+	}
+	lexer := newLexer()
+	stmts, err := parse(lexer, os.Stdin)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	for _, stmt := range stmts {
+		fmt.Println(stmt)
+	}
+}
+
+func parse(lexer *lexmachine.Lexer, fin io.Reader) (stmts []*Node, err error) {
+	defer func() {
+		if e := recover(); e != nil {
+			switch e.(type) {
+			case error:
+				err = e.(error)
+				stmts = nil
+			default:
+				panic(e)
+			}
+		}
+	}()
+	text, err := ioutil.ReadAll(fin)
+	if err != nil {
+		return nil, err
+	}
+	scanner, err := newGoLex(lexer, text)
+	if err != nil {
+		return nil, err
+	}
+	yyParse(scanner)
+	return scanner.stmts, nil
+}
diff --git a/examples/sensors-parser/sensors.conf b/examples/sensors-parser/sensors.conf
new file mode 100644
index 0000000..3c931a9
--- /dev/null
+++ b/examples/sensors-parser/sensors.conf
@@ -0,0 +1,4 @@
+# libsensors configuration file
+chip "lm78-*" "lm79-*" "lm80-*" "lm96080-*"
+    label temp1 "M/B Temp"
+chip "w83792d-*"
diff --git a/examples/sensors-parser/sensors.y b/examples/sensors-parser/sensors.y
new file mode 100644
index 0000000..60a2ae9
--- /dev/null
+++ b/examples/sensors-parser/sensors.y
@@ -0,0 +1,101 @@
+%{
+
+package main
+
+import (
+    "gitea.xintech.co/zhouzhihong/lexmachine"
+)
+
+%}
+
+%union{
+    token *lexmachine.Token
+    ast   *Node
+}
+
+%token	AT
+%token	PLUS
+%token	STAR
+%token	DASH
+%token	SLASH
+%token	BACKSLASH
+%token	CARROT
+%token	BACKTICK
+%token	COMMA
+%token	LPAREN
+%token	RPAREN
+%token	BUS
+%token	COMPUTE
+%token	CHIP
+%token	IGNORE
+%token	LABEL
+%token	SET
+%token	NUMBER
+%token	NAME
+%token	NEWLINE
+
+%% /* The grammar follows.  */
+
+Lines : Lines Line
+      | Line
+      ;
+
+Line : Stmt NEWLINE             { yylex.(*golex).stmts = append(yylex.(*golex).stmts, $1.ast) }
+     | NEWLINE
+     ;
+
+Stmt : Bus                      { $$.ast = $1.ast }
+     | Chip                     { $$.ast = $1.ast }
+     | Label                    { $$.ast = $1.ast }
+     | Compute                  { $$.ast = $1.ast }
+     | Ignore                   { $$.ast = $1.ast }
+     | Set                      { $$.ast = $1.ast }
+     ;
+
+Names : NAME Names              { $$.ast = $2.ast.PrependKid(NewNode("name", $1.token)) }
+      | NAME                    { $$.ast = NewNode("names", nil).AddKid(NewNode("name", $1.token)) }
+      ;
+
+Bus : BUS NAME NAME NAME        { $$.ast = NewNode("bus", $1.token).AddKid(NewNode("name", $2.token)).AddKid(NewNode("name", $3.token)).AddKid(NewNode("name", $4.token)) }
+    ;
+
+Chip : CHIP Names               { $$.ast = NewNode("chip", $1.token).AddKid($2.ast) }
+     ;
+
+Label : LABEL NAME NAME         { $$.ast = NewNode("label", $1.token).AddKid(NewNode("name", $2.token)).AddKid(NewNode("name", $3.token)) }
+      ;
+
+Compute : COMPUTE NAME Expr COMMA Expr
+                                { $$.ast = NewNode("compute", $1.token).AddKid(NewNode("name", $2.token)).AddKid($3.ast).AddKid($5.ast)  }
+        ;
+
+Ignore : IGNORE NAME            { $$.ast = NewNode("ignore", $1.token).AddKid(NewNode("name", $2.token)) }
+       ;
+
+Set : SET NAME Expr             { $$.ast = NewNode("set", $1.token).AddKid(NewNode("name", $2.token)).AddKid($3.ast) }
+    ;
+
+Expr : Expr PLUS Term           { $$.ast = NewNode("+", $2.token).AddKid($1.ast).AddKid($3.ast) }
+     | Expr DASH Term           { $$.ast = NewNode("-", $2.token).AddKid($1.ast).AddKid($3.ast) }
+     | Term                     { $$.ast = $1.ast }
+     ;
+
+Term : Term STAR Unary          { $$.ast = NewNode("*", $2.token).AddKid($1.ast).AddKid($3.ast) }
+     | Term SLASH Unary         { $$.ast = NewNode("/", $2.token).AddKid($1.ast).AddKid($3.ast) }
+     | Unary                    { $$.ast = $1.ast }
+     ;
+
+Unary : DASH Factor             { $$.ast = NewNode("negate", $1.token).AddKid($2.ast) }
+      | BACKTICK Factor         { $$.ast = NewNode("`", $1.token).AddKid($2.ast) }
+      | CARROT Factor           { $$.ast = NewNode("^", $1.token).AddKid($2.ast) }
+      | Factor                  { $$.ast = $1.ast }
+      ;
+
+Factor : NAME                   { $$.ast = NewNode("name", $1.token) }
+       | NUMBER                 { $$.ast = NewNode("number", $1.token) }
+       | AT                     { $$.ast = NewNode("@", $1.token) }
+       | LPAREN Expr RPAREN     { $$.ast = $2.ast }
+       ;
+
+;
+%%
diff --git a/examples/sensors-parser/sensors_golex.go b/examples/sensors-parser/sensors_golex.go
new file mode 100644
index 0000000..c3f8b3c
--- /dev/null
+++ b/examples/sensors-parser/sensors_golex.go
@@ -0,0 +1,115 @@
+// Package golex implements the same lexer as examples/sensors. However, it
+// shows how to conform to the goyacc's expected interface:
+//
+//	type yyLexer interface {
+//	   Lex(lval *yySymType) (tokenType int)
+//	   Error(message string)
+//	}
+//
+// You define yySymType. The yyLexer type is defined by the generated code
+// from goyacc. The tokenType is the token identifier. The expectation is
+// the token id's are shared between what is defined in this package and
+// the parser definition in parser.y.
+//
+// To generate the parser (and make this all work) run:
+//
+// go generate gitea.xintech.co/zhouzhihong/lexmachine/examples/golex
+//
+//go:generate sh -c "if [[ -z \"`which goyacc`\" ]]; then go get -u golang.org/x/tools/cmd/goyacc || exit 1; fi; if [[ -f y.go ]]; then rm y.*; fi; goyacc sensors.y"
+package main
+
+import (
+	"fmt"
+
+	"gitea.xintech.co/zhouzhihong/lexmachine"
+	"gitea.xintech.co/zhouzhihong/lexmachine/machines"
+)
+
+type golex struct {
+	*lexmachine.Scanner
+	stmts []*Node
+}
+
+// Construct a new golex from a lexer object and the text to parse.
+func newGoLex(lexer *lexmachine.Lexer, text []byte) (*golex, error) {
+	scan, err := lexer.Scanner(text)
+	if err != nil {
+		return nil, err
+	}
+	return &golex{Scanner: scan}, nil
+}
+
+// Lex implements yyLexer's interface for getting the next token. It returns the
+// token type as an integer. The tokens should be defined in the $parser.y file.
+// The actual number returned will be >= yyPrivate - 1 which is the range for
+// custom token names.
+func (g *golex) Lex(lval *yySymType) (tokenType int) {
+	s := g.Scanner
+	tok, err, eof := s.Next()
+	if err != nil {
+		g.Error(err.Error())
+	} else if eof {
+		return -1 // signals EOF to goyacc's yyParse
+	}
+	lval.token = tok.(*lexmachine.Token)
+	// To return the correct number for goyacc you must add yyPrivate - 1 to
+	// put the value into the correct range.
+	return lval.token.Type + yyPrivate - 1
+}
+
+// Error implements the error handling for if there is a parse error of any
+// kind. This implementation panics. There may be no better way to hand errors
+// from goyacc. I recommend you use defer ... recover() to handle this where
+// you call into the parser.
+func (l *golex) Error(message string) {
+	// is there a better way to handle this in the context of goyacc?
+	panic(fmt.Errorf(message))
+}
+
+// newLexer constructs the lexer for you. Only call this once.
+func newLexer() *lexmachine.Lexer {
+	// build the token map from yyToknames produced by goyacc from the %token
+	// directives.
+	tokmap := make(map[string]int)
+	for id, name := range yyToknames {
+		tokmap[name] = id
+	}
+	getToken := func(tokenType int) lexmachine.Action {
+		return func(s *lexmachine.Scanner, m *machines.Match) (interface{}, error) {
+			return s.Token(tokenType, string(m.Bytes), m), nil
+		}
+	}
+	skip := func(s *lexmachine.Scanner, m *machines.Match) (interface{}, error) {
+		return nil, nil
+	}
+	var lexer = lexmachine.NewLexer()
+	lexer.Add([]byte("@"), getToken(tokmap["AT"]))
+	lexer.Add([]byte(`\+`), getToken(tokmap["PLUS"]))
+	lexer.Add([]byte(`\*`), getToken(tokmap["STAR"]))
+	lexer.Add([]byte("-"), getToken(tokmap["DASH"]))
+	lexer.Add([]byte("/"), getToken(tokmap["SLASH"]))
+	lexer.Add([]byte("\\"), getToken(tokmap["BACKSLASH"]))
+	lexer.Add([]byte(`\^`), getToken(tokmap["CARROT"]))
+	lexer.Add([]byte("`"), getToken(tokmap["BACKTICK"]))
+	lexer.Add([]byte(","), getToken(tokmap["COMMA"]))
+	lexer.Add([]byte(`\(`), getToken(tokmap["LPAREN"]))
+	lexer.Add([]byte(`\)`), getToken(tokmap["RPAREN"]))
+	lexer.Add([]byte("bus"), getToken(tokmap["BUS"]))
+	lexer.Add([]byte("chip"), getToken(tokmap["CHIP"]))
+	lexer.Add([]byte("label"), getToken(tokmap["LABEL"]))
+	lexer.Add([]byte("compute"), getToken(tokmap["COMPUTE"]))
+	lexer.Add([]byte("ignore"), getToken(tokmap["IGNORE"]))
+	lexer.Add([]byte("set"), getToken(tokmap["SET"]))
+	lexer.Add([]byte(`[0-9]*\.?[0-9]+`), getToken(tokmap["NUMBER"]))
+	lexer.Add([]byte(`[a-zA-Z_][a-zA-Z0-9_]*`), getToken(tokmap["NAME"]))
+	lexer.Add([]byte(`"[^"]*"`), getToken(tokmap["NAME"]))
+	lexer.Add([]byte(`\\\n`), skip) // skip backslash newline
+	lexer.Add([]byte(`\n`), getToken(tokmap["NEWLINE"]))
+	lexer.Add([]byte(`#[^\n]*`), skip)
+	lexer.Add([]byte(` |\t`), skip)
+	err := lexer.Compile()
+	if err != nil {
+		panic(err)
+	}
+	return lexer
+}
diff --git a/examples/sensors-parser/y.go b/examples/sensors-parser/y.go
new file mode 100644
index 0000000..4de7824
--- /dev/null
+++ b/examples/sensors-parser/y.go
@@ -0,0 +1,665 @@
+//line sensors.y:2
+package main
+
+import (
+	__yyfmt__ "fmt"
+
+	"gitea.xintech.co/zhouzhihong/lexmachine"
+) //line sensors.y:3
+
+//line sensors.y:11
+type yySymType struct {
+	yys   int
+	token *lexmachine.Token
+	ast   *Node
+}
+
+const AT = 57346
+const PLUS = 57347
+const STAR = 57348
+const DASH = 57349
+const SLASH = 57350
+const BACKSLASH = 57351
+const CARROT = 57352
+const BACKTICK = 57353
+const COMMA = 57354
+const LPAREN = 57355
+const RPAREN = 57356
+const BUS = 57357
+const COMPUTE = 57358
+const CHIP = 57359
+const IGNORE = 57360
+const LABEL = 57361
+const SET = 57362
+const NUMBER = 57363
+const NAME = 57364
+const NEWLINE = 57365
+
+var yyToknames = [...]string{
+	"$end",
+	"error",
+	"$unk",
+	"AT",
+	"PLUS",
+	"STAR",
+	"DASH",
+	"SLASH",
+	"BACKSLASH",
+	"CARROT",
+	"BACKTICK",
+	"COMMA",
+	"LPAREN",
+	"RPAREN",
+	"BUS",
+	"COMPUTE",
+	"CHIP",
+	"IGNORE",
+	"LABEL",
+	"SET",
+	"NUMBER",
+	"NAME",
+	"NEWLINE",
+}
+var yyStatenames = [...]string{}
+
+const yyEofCode = 1
+const yyErrCode = 2
+const yyInitialStackSize = 16
+
+//line sensors.y:101
+
+//line yacctab:1
+var yyExca = [...]int{
+	-1, 1,
+	1, -1,
+	-2, 0,
+}
+
+const yyPrivate = 57344
+
+const yyLast = 64
+
+var yyAct = [...]int{
+
+	31, 29, 35, 18, 30, 11, 14, 12, 15, 13,
+	16, 38, 20, 4, 32, 38, 41, 34, 33, 28,
+	39, 21, 26, 25, 39, 24, 23, 40, 37, 36,
+	22, 19, 37, 36, 27, 47, 48, 49, 45, 10,
+	46, 50, 9, 43, 51, 44, 54, 55, 52, 53,
+	8, 43, 56, 44, 43, 2, 44, 17, 42, 7,
+	6, 5, 3, 1,
+}
+var yyPact = [...]int{
+
+	-10, -10, -1000, -20, -1000, -1000, -1000, -1000, -1000, -1000,
+	-1000, 9, -1, 8, 4, 3, 1, -1000, -1000, 0,
+	-1000, -1, -3, 7, -1000, 7, -6, -1000, -1000, 46,
+	32, -1000, 11, 11, 11, -1000, -1000, -1000, -1000, 7,
+	49, -1000, 7, 7, 7, 7, 7, -1000, -1000, -1000,
+	38, 49, 32, 32, -1000, -1000, -1000,
+}
+var yyPgo = [...]int{
+
+	0, 63, 55, 62, 61, 60, 59, 50, 42, 39,
+	12, 1, 4, 0, 2,
+}
+var yyR1 = [...]int{
+
+	0, 1, 1, 2, 2, 3, 3, 3, 3, 3,
+	3, 10, 10, 4, 5, 6, 7, 8, 9, 11,
+	11, 11, 12, 12, 12, 13, 13, 13, 13, 14,
+	14, 14, 14,
+}
+var yyR2 = [...]int{
+
+	0, 2, 1, 2, 1, 1, 1, 1, 1, 1,
+	1, 2, 1, 4, 2, 3, 5, 2, 3, 3,
+	3, 1, 3, 3, 1, 2, 2, 2, 1, 1,
+	1, 1, 3,
+}
+var yyChk = [...]int{
+
+	-1000, -1, -2, -3, 23, -4, -5, -6, -7, -8,
+	-9, 15, 17, 19, 16, 18, 20, -2, 23, 22,
+	-10, 22, 22, 22, 22, 22, 22, -10, 22, -11,
+	-12, -13, 7, 11, 10, -14, 22, 21, 4, 13,
+	-11, 22, 12, 5, 7, 6, 8, -14, -14, -14,
+	-11, -11, -12, -12, -13, -13, 14,
+}
+var yyDef = [...]int{
+
+	0, -2, 2, 0, 4, 5, 6, 7, 8, 9,
+	10, 0, 0, 0, 0, 0, 0, 1, 3, 0,
+	14, 12, 0, 0, 17, 0, 0, 11, 15, 0,
+	21, 24, 0, 0, 0, 28, 29, 30, 31, 0,
+	18, 13, 0, 0, 0, 0, 0, 25, 26, 27,
+	0, 16, 19, 20, 22, 23, 32,
+}
+var yyTok1 = [...]int{
+
+	1,
+}
+var yyTok2 = [...]int{
+
+	2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
+	12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
+	22, 23,
+}
+var yyTok3 = [...]int{
+	0,
+}
+
+var yyErrorMessages = [...]struct {
+	state int
+	token int
+	msg   string
+}{}
+
+//line yaccpar:1
+
+/*	parser for yacc output	*/
+
+var (
+	yyDebug        = 0
+	yyErrorVerbose = false
+)
+
+type yyLexer interface {
+	Lex(lval *yySymType) int
+	Error(s string)
+}
+
+type yyParser interface {
+	Parse(yyLexer) int
+	Lookahead() int
+}
+
+type yyParserImpl struct {
+	lval  yySymType
+	stack [yyInitialStackSize]yySymType
+	char  int
+}
+
+func (p *yyParserImpl) Lookahead() int {
+	return p.char
+}
+
+func yyNewParser() yyParser {
+	return &yyParserImpl{}
+}
+
+const yyFlag = -1000
+
+func yyTokname(c int) string {
+	if c >= 1 && c-1 < len(yyToknames) {
+		if yyToknames[c-1] != "" {
+			return yyToknames[c-1]
+		}
+	}
+	return __yyfmt__.Sprintf("tok-%v", c)
+}
+
+func yyStatname(s int) string {
+	if s >= 0 && s < len(yyStatenames) {
+		if yyStatenames[s] != "" {
+			return yyStatenames[s]
+		}
+	}
+	return __yyfmt__.Sprintf("state-%v", s)
+}
+
+func yyErrorMessage(state, lookAhead int) string {
+	const TOKSTART = 4
+
+	if !yyErrorVerbose {
+		return "syntax error"
+	}
+
+	for _, e := range yyErrorMessages {
+		if e.state == state && e.token == lookAhead {
+			return "syntax error: " + e.msg
+		}
+	}
+
+	res := "syntax error: unexpected " + yyTokname(lookAhead)
+
+	// To match Bison, suggest at most four expected tokens.
+	expected := make([]int, 0, 4)
+
+	// Look for shiftable tokens.
+	base := yyPact[state]
+	for tok := TOKSTART; tok-1 < len(yyToknames); tok++ {
+		if n := base + tok; n >= 0 && n < yyLast && yyChk[yyAct[n]] == tok {
+			if len(expected) == cap(expected) {
+				return res
+			}
+			expected = append(expected, tok)
+		}
+	}
+
+	if yyDef[state] == -2 {
+		i := 0
+		for yyExca[i] != -1 || yyExca[i+1] != state {
+			i += 2
+		}
+
+		// Look for tokens that we accept or reduce.
+		for i += 2; yyExca[i] >= 0; i += 2 {
+			tok := yyExca[i]
+			if tok < TOKSTART || yyExca[i+1] == 0 {
+				continue
+			}
+			if len(expected) == cap(expected) {
+				return res
+			}
+			expected = append(expected, tok)
+		}
+
+		// If the default action is to accept or reduce, give up.
+		if yyExca[i+1] != 0 {
+			return res
+		}
+	}
+
+	for i, tok := range expected {
+		if i == 0 {
+			res += ", expecting "
+		} else {
+			res += " or "
+		}
+		res += yyTokname(tok)
+	}
+	return res
+}
+
+func yylex1(lex yyLexer, lval *yySymType) (char, token int) {
+	token = 0
+	char = lex.Lex(lval)
+	if char <= 0 {
+		token = yyTok1[0]
+		goto out
+	}
+	if char < len(yyTok1) {
+		token = yyTok1[char]
+		goto out
+	}
+	if char >= yyPrivate {
+		if char < yyPrivate+len(yyTok2) {
+			token = yyTok2[char-yyPrivate]
+			goto out
+		}
+	}
+	for i := 0; i < len(yyTok3); i += 2 {
+		token = yyTok3[i+0]
+		if token == char {
+			token = yyTok3[i+1]
+			goto out
+		}
+	}
+
+out:
+	if token == 0 {
+		token = yyTok2[1] /* unknown char */
+	}
+	if yyDebug >= 3 {
+		__yyfmt__.Printf("lex %s(%d)\n", yyTokname(token), uint(char))
+	}
+	return char, token
+}
+
+func yyParse(yylex yyLexer) int {
+	return yyNewParser().Parse(yylex)
+}
+
+func (yyrcvr *yyParserImpl) Parse(yylex yyLexer) int {
+	var yyn int
+	var yyVAL yySymType
+	var yyDollar []yySymType
+	_ = yyDollar // silence set and not used
+	yyS := yyrcvr.stack[:]
+
+	Nerrs := 0   /* number of errors */
+	Errflag := 0 /* error recovery flag */
+	yystate := 0
+	yyrcvr.char = -1
+	yytoken := -1 // yyrcvr.char translated into internal numbering
+	defer func() {
+		// Make sure we report no lookahead when not parsing.
+		yystate = -1
+		yyrcvr.char = -1
+		yytoken = -1
+	}()
+	yyp := -1
+	goto yystack
+
+ret0:
+	return 0
+
+ret1:
+	return 1
+
+yystack:
+	/* put a state and value onto the stack */
+	if yyDebug >= 4 {
+		__yyfmt__.Printf("char %v in %v\n", yyTokname(yytoken), yyStatname(yystate))
+	}
+
+	yyp++
+	if yyp >= len(yyS) {
+		nyys := make([]yySymType, len(yyS)*2)
+		copy(nyys, yyS)
+		yyS = nyys
+	}
+	yyS[yyp] = yyVAL
+	yyS[yyp].yys = yystate
+
+yynewstate:
+	yyn = yyPact[yystate]
+	if yyn <= yyFlag {
+		goto yydefault /* simple state */
+	}
+	if yyrcvr.char < 0 {
+		yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval)
+	}
+	yyn += yytoken
+	if yyn < 0 || yyn >= yyLast {
+		goto yydefault
+	}
+	yyn = yyAct[yyn]
+	if yyChk[yyn] == yytoken { /* valid shift */
+		yyrcvr.char = -1
+		yytoken = -1
+		yyVAL = yyrcvr.lval
+		yystate = yyn
+		if Errflag > 0 {
+			Errflag--
+		}
+		goto yystack
+	}
+
+yydefault:
+	/* default state action */
+	yyn = yyDef[yystate]
+	if yyn == -2 {
+		if yyrcvr.char < 0 {
+			yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval)
+		}
+
+		/* look through exception table */
+		xi := 0
+		for {
+			if yyExca[xi+0] == -1 && yyExca[xi+1] == yystate {
+				break
+			}
+			xi += 2
+		}
+		for xi += 2; ; xi += 2 {
+			yyn = yyExca[xi+0]
+			if yyn < 0 || yyn == yytoken {
+				break
+			}
+		}
+		yyn = yyExca[xi+1]
+		if yyn < 0 {
+			goto ret0
+		}
+	}
+	if yyn == 0 {
+		/* error ... attempt to resume parsing */
+		switch Errflag {
+		case 0: /* brand new error */
+			yylex.Error(yyErrorMessage(yystate, yytoken))
+			Nerrs++
+			if yyDebug >= 1 {
+				__yyfmt__.Printf("%s", yyStatname(yystate))
+				__yyfmt__.Printf(" saw %s\n", yyTokname(yytoken))
+			}
+			fallthrough
+
+		case 1, 2: /* incompletely recovered error ... try again */
+			Errflag = 3
+
+			/* find a state where "error" is a legal shift action */
+			for yyp >= 0 {
+				yyn = yyPact[yyS[yyp].yys] + yyErrCode
+				if yyn >= 0 && yyn < yyLast {
+					yystate = yyAct[yyn] /* simulate a shift of "error" */
+					if yyChk[yystate] == yyErrCode {
+						goto yystack
+					}
+				}
+
+				/* the current p has no shift on "error", pop stack */
+				if yyDebug >= 2 {
+					__yyfmt__.Printf("error recovery pops state %d\n", yyS[yyp].yys)
+				}
+				yyp--
+			}
+			/* there is no state on the stack with an error shift ... abort */
+			goto ret1
+
+		case 3: /* no shift yet; clobber input char */
+			if yyDebug >= 2 {
+				__yyfmt__.Printf("error recovery discards %s\n", yyTokname(yytoken))
+			}
+			if yytoken == yyEofCode {
+				goto ret1
+			}
+			yyrcvr.char = -1
+			yytoken = -1
+			goto yynewstate /* try again in the same state */
+		}
+	}
+
+	/* reduction by production yyn */
+	if yyDebug >= 2 {
+		__yyfmt__.Printf("reduce %v in:\n\t%v\n", yyn, yyStatname(yystate))
+	}
+
+	yynt := yyn
+	yypt := yyp
+	_ = yypt // guard against "declared and not used"
+
+	yyp -= yyR2[yyn]
+	// yyp is now the index of $0. Perform the default action. Iff the
+	// reduced production is ε, $1 is possibly out of range.
+	if yyp+1 >= len(yyS) {
+		nyys := make([]yySymType, len(yyS)*2)
+		copy(nyys, yyS)
+		yyS = nyys
+	}
+	yyVAL = yyS[yyp+1]
+
+	/* consult goto table to find next state */
+	yyn = yyR1[yyn]
+	yyg := yyPgo[yyn]
+	yyj := yyg + yyS[yyp].yys + 1
+
+	if yyj >= yyLast {
+		yystate = yyAct[yyg]
+	} else {
+		yystate = yyAct[yyj]
+		if yyChk[yystate] != -yyn {
+			yystate = yyAct[yyg]
+		}
+	}
+	// dummy call; replaced with literal code
+	switch yynt {
+
+	case 3:
+		yyDollar = yyS[yypt-2 : yypt+1]
+		//line sensors.y:43
+		{
+			yylex.(*golex).stmts = append(yylex.(*golex).stmts, yyDollar[1].ast)
+		}
+	case 5:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:47
+		{
+			yyVAL.ast = yyDollar[1].ast
+		}
+	case 6:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:48
+		{
+			yyVAL.ast = yyDollar[1].ast
+		}
+	case 7:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:49
+		{
+			yyVAL.ast = yyDollar[1].ast
+		}
+	case 8:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:50
+		{
+			yyVAL.ast = yyDollar[1].ast
+		}
+	case 9:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:51
+		{
+			yyVAL.ast = yyDollar[1].ast
+		}
+	case 10:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:52
+		{
+			yyVAL.ast = yyDollar[1].ast
+		}
+	case 11:
+		yyDollar = yyS[yypt-2 : yypt+1]
+		//line sensors.y:55
+		{
+			yyVAL.ast = yyDollar[2].ast.PrependKid(NewNode("name", yyDollar[1].token))
+		}
+	case 12:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:56
+		{
+			yyVAL.ast = NewNode("names", nil).AddKid(NewNode("name", yyDollar[1].token))
+		}
+	case 13:
+		yyDollar = yyS[yypt-4 : yypt+1]
+		//line sensors.y:59
+		{
+			yyVAL.ast = NewNode("bus", yyDollar[1].token).AddKid(NewNode("name", yyDollar[2].token)).AddKid(NewNode("name", yyDollar[3].token)).AddKid(NewNode("name", yyDollar[4].token))
+		}
+	case 14:
+		yyDollar = yyS[yypt-2 : yypt+1]
+		//line sensors.y:62
+		{
+			yyVAL.ast = NewNode("chip", yyDollar[1].token).AddKid(yyDollar[2].ast)
+		}
+	case 15:
+		yyDollar = yyS[yypt-3 : yypt+1]
+		//line sensors.y:65
+		{
+			yyVAL.ast = NewNode("label", yyDollar[1].token).AddKid(NewNode("name", yyDollar[2].token)).AddKid(NewNode("name", yyDollar[3].token))
+		}
+	case 16:
+		yyDollar = yyS[yypt-5 : yypt+1]
+		//line sensors.y:69
+		{
+			yyVAL.ast = NewNode("compute", yyDollar[1].token).AddKid(NewNode("name", yyDollar[2].token)).AddKid(yyDollar[3].ast).AddKid(yyDollar[5].ast)
+		}
+	case 17:
+		yyDollar = yyS[yypt-2 : yypt+1]
+		//line sensors.y:72
+		{
+			yyVAL.ast = NewNode("ignore", yyDollar[1].token).AddKid(NewNode("name", yyDollar[2].token))
+		}
+	case 18:
+		yyDollar = yyS[yypt-3 : yypt+1]
+		//line sensors.y:75
+		{
+			yyVAL.ast = NewNode("set", yyDollar[1].token).AddKid(NewNode("name", yyDollar[2].token)).AddKid(yyDollar[3].ast)
+		}
+	case 19:
+		yyDollar = yyS[yypt-3 : yypt+1]
+		//line sensors.y:78
+		{
+			yyVAL.ast = NewNode("+", yyDollar[2].token).AddKid(yyDollar[1].ast).AddKid(yyDollar[3].ast)
+		}
+	case 20:
+		yyDollar = yyS[yypt-3 : yypt+1]
+		//line sensors.y:79
+		{
+			yyVAL.ast = NewNode("-", yyDollar[2].token).AddKid(yyDollar[1].ast).AddKid(yyDollar[3].ast)
+		}
+	case 21:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:80
+		{
+			yyVAL.ast = yyDollar[1].ast
+		}
+	case 22:
+		yyDollar = yyS[yypt-3 : yypt+1]
+		//line sensors.y:83
+		{
+			yyVAL.ast = NewNode("*", yyDollar[2].token).AddKid(yyDollar[1].ast).AddKid(yyDollar[3].ast)
+		}
+	case 23:
+		yyDollar = yyS[yypt-3 : yypt+1]
+		//line sensors.y:84
+		{
+			yyVAL.ast = NewNode("/", yyDollar[2].token).AddKid(yyDollar[1].ast).AddKid(yyDollar[3].ast)
+		}
+	case 24:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:85
+		{
+			yyVAL.ast = yyDollar[1].ast
+		}
+	case 25:
+		yyDollar = yyS[yypt-2 : yypt+1]
+		//line sensors.y:88
+		{
+			yyVAL.ast = NewNode("negate", yyDollar[1].token).AddKid(yyDollar[2].ast)
+		}
+	case 26:
+		yyDollar = yyS[yypt-2 : yypt+1]
+		//line sensors.y:89
+		{
+			yyVAL.ast = NewNode("`", yyDollar[1].token).AddKid(yyDollar[2].ast)
+		}
+	case 27:
+		yyDollar = yyS[yypt-2 : yypt+1]
+		//line sensors.y:90
+		{
+			yyVAL.ast = NewNode("^", yyDollar[1].token).AddKid(yyDollar[2].ast)
+		}
+	case 28:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:91
+		{
+			yyVAL.ast = yyDollar[1].ast
+		}
+	case 29:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:94
+		{
+			yyVAL.ast = NewNode("name", yyDollar[1].token)
+		}
+	case 30:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:95
+		{
+			yyVAL.ast = NewNode("number", yyDollar[1].token)
+		}
+	case 31:
+		yyDollar = yyS[yypt-1 : yypt+1]
+		//line sensors.y:96
+		{
+			yyVAL.ast = NewNode("@", yyDollar[1].token)
+		}
+	case 32:
+		yyDollar = yyS[yypt-3 : yypt+1]
+		//line sensors.y:97
+		{
+			yyVAL.ast = yyDollar[2].ast
+		}
+	}
+	goto yystack /* stack new state and value */
+}
diff --git a/examples/sensors/main.go b/examples/sensors/main.go
new file mode 100644
index 0000000..5f0443e
--- /dev/null
+++ b/examples/sensors/main.go
@@ -0,0 +1,130 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"gitea.xintech.co/zhouzhihong/lexmachine"
+	"gitea.xintech.co/zhouzhihong/lexmachine/machines"
+	"github.com/timtadh/getopt"
+)
+
+var tokens = []string{
+	"AT", "PLUS", "STAR", "DASH", "SLASH", "BACKSLASH", "CARROT", "BACKTICK", "COMMA", "LPAREN", "RPAREN",
+	"BUS", "COMPUTE", "CHIP", "IGNORE", "LABEL", "SET", "NUMBER", "NAME",
+	"COMMENT", "SPACE",
+}
+var tokmap map[string]int
+var lexer *lexmachine.Lexer
+
+func init() {
+	tokmap = make(map[string]int)
+	for id, name := range tokens {
+		tokmap[name] = id
+	}
+}
+
+func newLexer(dfa bool) *lexmachine.Lexer {
+	getToken := func(tokenType int) lexmachine.Action {
+		return func(s *lexmachine.Scanner, m *machines.Match) (interface{}, error) {
+			return s.Token(tokenType, string(m.Bytes), m), nil
+		}
+	}
+	var lexer = lexmachine.NewLexer()
+	lexer.Add([]byte("@"), getToken(tokmap["AT"]))
+	lexer.Add([]byte(`\+`), getToken(tokmap["PLUS"]))
+	lexer.Add([]byte(`\*`), getToken(tokmap["STAR"]))
+	lexer.Add([]byte("-"), getToken(tokmap["DASH"]))
+	lexer.Add([]byte("/"), getToken(tokmap["SLASH"]))
+	lexer.Add([]byte("\\"), getToken(tokmap["BACKSLASH"]))
+	lexer.Add([]byte(`\^`), getToken(tokmap["CARROT"]))
+	lexer.Add([]byte("`"), getToken(tokmap["BACKTICK"]))
+	lexer.Add([]byte(","), getToken(tokmap["COMMA"]))
+	lexer.Add([]byte(`\(`), getToken(tokmap["LPAREN"]))
+	lexer.Add([]byte(`\)`), getToken(tokmap["RPAREN"]))
+	lexer.Add([]byte("bus"), getToken(tokmap["BUS"]))
+	lexer.Add([]byte("chip"), getToken(tokmap["CHIP"]))
+	lexer.Add([]byte("label"), getToken(tokmap["LABEL"]))
+	lexer.Add([]byte("compute"), getToken(tokmap["COMPUTE"]))
+	lexer.Add([]byte("ignore"), getToken(tokmap["IGNORE"]))
+	lexer.Add([]byte("set"), getToken(tokmap["SET"]))
+	lexer.Add([]byte(`[0-9]*\.?[0-9]+`), getToken(tokmap["NUMBER"]))
+	lexer.Add([]byte(`[a-zA-Z_][a-zA-Z0-9_]*`), getToken(tokmap["NAME"]))
+	lexer.Add([]byte(`"[^"]*"`), getToken(tokmap["NAME"]))
+	lexer.Add([]byte(`#[^\n]*`), getToken(tokmap["COMMENT"]))
+	lexer.Add([]byte(`\s+`), getToken(tokmap["SPACE"]))
+	var err error
+	if dfa {
+		err = lexer.CompileDFA()
+	} else {
+		err = lexer.CompileNFA()
+	}
+	if err != nil {
+		panic(err)
+	}
+	return lexer
+}
+
+func scan(text []byte) error {
+	scanner, err := lexer.Scanner(text)
+	if err != nil {
+		return err
+	}
+	for tk, err, eof := scanner.Next(); !eof; tk, err, eof = scanner.Next() {
+		if ui, is := err.(*machines.UnconsumedInput); ui != nil && is {
+			scanner.TC = ui.FailTC
+			log.Printf("skipping %v", ui)
+		} else if err != nil {
+			return err
+		} else {
+			if false {
+				fmt.Println(tk)
+			}
+		}
+	}
+	return nil
+}
+
+func main() {
+	short := "hdn"
+	long := []string{
+		"help",
+		"dfa",
+		"nfa",
+	}
+
+	_, optargs, err := getopt.GetOpt(os.Args[1:], short, long)
+	if err != nil {
+		log.Print(err)
+		log.Println("help")
+		os.Exit(1)
+	}
+
+	dfa := false
+	for _, oa := range optargs {
+		switch oa.Opt() {
+		case "-h", "--help":
+			fmt.Println("Tokenizes the standard input 1000 times")
+			fmt.Println("Must supply either --nfa or --dfa. try cat /etc/sensors*.conf | sensors --nfa")
+			os.Exit(0)
+		case "-d", "--dfa":
+			dfa = true
+		case "-n", "--nfa":
+			dfa = false
+		}
+	}
+	lexer = newLexer(dfa)
+
+	text, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for i := 0; i < 1000; i++ {
+		err = scan(text)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}
diff --git a/frontend/ast.go b/frontend/ast.go
index a10ba09..dd9368a 100644
--- a/frontend/ast.go
+++ b/frontend/ast.go
@@ -140,8 +140,8 @@ func (c *Concat) String() string {
 
 // Range matches byte ranges From-To inclusive
 type Range struct {
-	From rune
-	To   rune
+	From byte
+	To   byte
 }
 
 // Children returns a list of the child nodes
@@ -160,7 +160,7 @@ func (r *Range) String() string {
 
 // Character matches a single byte
 type Character struct {
-	Char rune
+	Char byte
 }
 
 // Children returns a list of the child nodes
@@ -172,7 +172,7 @@ func (c *Character) Children() []AST {
 func (c *Character) String() string {
 	return fmt.Sprintf(
 		"(Character %s)",
-		string([]rune{c.Char}),
+		string([]byte{c.Char}),
 	)
 }
 
@@ -248,7 +248,7 @@ func NewConcat(char, concat AST) AST {
 }
 
 // NewCharacter constructs a character
-func NewCharacter(b rune) *Character {
+func NewCharacter(b byte) *Character {
 	return &Character{b}
 }
 
@@ -258,7 +258,7 @@ func NewAny() *Range {
 }
 
 // NewRange constructs a range operator
-func NewRange(from, to rune) *Range {
+func NewRange(from, to byte) *Range {
 	if from <= to {
 		return &Range{From: from, To: to}
 	}
diff --git a/frontend/desugar.go b/frontend/desugar.go
index cdee56e..35b4d5d 100644
--- a/frontend/desugar.go
+++ b/frontend/desugar.go
@@ -30,7 +30,7 @@ func DesugarRanges(ast AST) AST {
 	case *Range:
 		chars := make([]*Character, 0, n.To-n.From+1)
 		for i := int(n.From); i <= int(n.To); i++ {
-			chars = append(chars, NewCharacter(rune(i)))
+			chars = append(chars, NewCharacter(byte(i)))
 		}
 		if len(chars) <= 0 {
 			panic(fmt.Errorf("Empty, unmatchable range: %v", n))
diff --git a/frontend/desugar_test.go b/frontend/desugar_test.go
index ae2ec34..e44ceb5 100644
--- a/frontend/desugar_test.go
+++ b/frontend/desugar_test.go
@@ -1,14 +1,11 @@
 package frontend
 
-import (
-	"testing"
-
-	"github.com/timtadh/data-structures/test"
-)
+import "testing"
+import "github.com/timtadh/data-structures/test"
 
 func TestDesugarRanges_any(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("."))
+	ast, err := Parse([]byte("."))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -25,7 +22,7 @@ func TestDesugarRanges_any(x *testing.T) {
 
 func TestDesugarRanges(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("(([a-z]+[A-Z])*[0-9])?wizard"))
+	ast, err := Parse([]byte("(([a-z]+[A-Z])*[0-9])?wizard"))
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/frontend/frontend_test.go b/frontend/frontend_test.go
index c99a4f2..de345c9 100644
--- a/frontend/frontend_test.go
+++ b/frontend/frontend_test.go
@@ -10,7 +10,7 @@ import (
 
 func TestParse(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("ab(a|c|d)?we*\\\\\\[\\..[s-f]+|qyx"))
+	ast, err := Parse([]byte("ab(a|c|d)?we*\\\\\\[\\..[s-f]+|qyx"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -23,12 +23,12 @@ func TestParse(x *testing.T) {
 }
 
 func tMatch(program inst.Slice, text string, t *test.T) {
-	expected := []machines.Match{{len(program) - 1, 0, 1, 1, 1, len(text), []rune(text)}}
+	expected := []machines.Match{{len(program) - 1, 0, 1, 1, 1, len(text), []byte(text)}}
 	if expected[0].EndColumn == 0 {
 		expected[0].EndColumn = 1
 	}
 	i := 0
-	scan := machines.LexerEngine(program, []rune(text))
+	scan := machines.LexerEngine(program, []byte(text))
 	for tc, m, err, scan := scan(0); scan != nil; tc, m, err, scan = scan(tc) {
 		t.Log("match", m)
 		if err != nil {
@@ -42,7 +42,7 @@ func tMatch(program inst.Slice, text string, t *test.T) {
 }
 
 func tNoMatch(program inst.Slice, text string, t *test.T) {
-	scan := machines.LexerEngine(program, []rune(text))
+	scan := machines.LexerEngine(program, []byte(text))
 	for tc, m, err, scan := scan(0); scan != nil; tc, m, err, scan = scan(tc) {
 		if err == nil {
 			t.Errorf("expected no match got %q, for %q", m, text)
@@ -54,7 +54,7 @@ func tNoMatch(program inst.Slice, text string, t *test.T) {
 
 func TestParseConcatAlts(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("A|((C|D|E)(F|G)(H|I)B)"))
+	ast, err := Parse([]byte("A|((C|D|E)(F|G)(H|I)B)"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -86,7 +86,7 @@ func TestParseConcatAlts(x *testing.T) {
 
 func TestParseConcatAltMaybes(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("((A?)?|(B|C))(D|E?)"))
+	ast, err := Parse([]byte("((A?)?|(B|C))(D|E?)"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -117,7 +117,7 @@ func TestParseConcatAltMaybes(x *testing.T) {
 
 func TestParseConcatAltPlus(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("(A|(B|C))+(D|E?)"))
+	ast, err := Parse([]byte("(A|(B|C))+(D|E?)"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -141,7 +141,7 @@ func TestParseConcatAltPlus(x *testing.T) {
 
 func TestParseAltOps(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("a|b"))
+	ast, err := Parse([]byte("a|b"))
 	t.AssertNil(err)
 	parsed := "(Match (Concat (Alternation (Character a), (Character b)), (EOS)))"
 	if ast.String() != parsed {
@@ -150,7 +150,7 @@ func TestParseAltOps(x *testing.T) {
 		t.Error("Did not parse correctly")
 	}
 
-	ast, err = Parse([]rune("a?|b"))
+	ast, err = Parse([]byte("a?|b"))
 	t.AssertNil(err)
 	parsed = "(Match (Concat (Alternation (? (Character a)), (Character b)), (EOS)))"
 	if ast.String() != parsed {
@@ -159,7 +159,7 @@ func TestParseAltOps(x *testing.T) {
 		t.Error("Did not parse correctly")
 	}
 
-	ast, err = Parse([]rune("a|b?"))
+	ast, err = Parse([]byte("a|b?"))
 	t.AssertNil(err)
 	parsed = "(Match (Concat (Alternation (Character a), (? (Character b))), (EOS)))"
 	if ast.String() != parsed {
@@ -168,7 +168,7 @@ func TestParseAltOps(x *testing.T) {
 		t.Error("Did not parse correctly")
 	}
 
-	ast, err = Parse([]rune("a?|b?"))
+	ast, err = Parse([]byte("a?|b?"))
 	t.AssertNil(err)
 	parsed = "(Match (Concat (Alternation (? (Character a)), (? (Character b))), (EOS)))"
 	if ast.String() != parsed {
@@ -180,7 +180,7 @@ func TestParseAltOps(x *testing.T) {
 
 func TestChainedOps(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("A?+*B*?+C+*?(x+?)**"))
+	ast, err := Parse([]byte("A?+*B*?+C+*?(x+?)**"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -194,7 +194,7 @@ func TestChainedOps(x *testing.T) {
 
 func TestParseConcatAltStar(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("(A|[C-G])*(X|Y?)"))
+	ast, err := Parse([]byte("(A|[C-G])*(X|Y?)"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -222,7 +222,7 @@ func TestParseConcatAltStar(x *testing.T) {
 
 func TestIdent(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("([a-z]|[A-Z])([a-z]|[A-Z]|[0-9]|_)*"))
+	ast, err := Parse([]byte("([a-z]|[A-Z])([a-z]|[A-Z]|[0-9]|_)*"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -246,7 +246,7 @@ func TestIdent(x *testing.T) {
 
 func TestDigitClass(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("\\d+"))
+	ast, err := Parse([]byte("\\d+"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -267,7 +267,7 @@ func TestDigitClass(x *testing.T) {
 
 func TestNotDigitClass(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("\\D+"))
+	ast, err := Parse([]byte("\\D+"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -288,7 +288,7 @@ func TestNotDigitClass(x *testing.T) {
 
 func TestSpaceClass(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("\\s+"))
+	ast, err := Parse([]byte("\\s+"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -309,7 +309,7 @@ func TestSpaceClass(x *testing.T) {
 
 func TestNoSpaceClass(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("\\S+"))
+	ast, err := Parse([]byte("\\S+"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -330,7 +330,7 @@ func TestNoSpaceClass(x *testing.T) {
 
 func TestWordClass(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("\\w+"))
+	ast, err := Parse([]byte("\\w+"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -352,7 +352,7 @@ func TestWordClass(x *testing.T) {
 
 func TestNoWordClass(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("\\W+"))
+	ast, err := Parse([]byte("\\W+"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -373,7 +373,7 @@ func TestNoWordClass(x *testing.T) {
 
 func TestMultiRangeClasses(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("([a-zA-Z])([a-zA-Z0-9_])*"))
+	ast, err := Parse([]byte("([a-zA-Z])([a-zA-Z0-9_])*"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -397,7 +397,7 @@ func TestMultiRangeClasses(x *testing.T) {
 
 func TestMultiRangeClasses2(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("([\\._/:a-zA-Z]+):\"(.+)\""))
+	ast, err := Parse([]byte("([\\._/:a-zA-Z]+):\"(.+)\""))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -418,7 +418,7 @@ func TestMultiRangeClasses2(x *testing.T) {
 
 func TestInvertRangeClasses1(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("[^abcd]+"))
+	ast, err := Parse([]byte("[^abcd]+"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -445,7 +445,7 @@ func TestInvertRangeClasses1(x *testing.T) {
 
 func TestInvertRangeClasses2(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("[^a-d]+"))
+	ast, err := Parse([]byte("[^a-d]+"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -472,7 +472,7 @@ func TestInvertRangeClasses2(x *testing.T) {
 
 func TestInvertRangeClasses3(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("[^a-dxyz]+"))
+	ast, err := Parse([]byte("[^a-dxyz]+"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -502,7 +502,7 @@ func TestInvertRangeClasses3(x *testing.T) {
 
 func TestLineComment(x *testing.T) {
 	t := (*test.T)(x)
-	ast, err := Parse([]rune("//[^\n]*"))
+	ast, err := Parse([]byte("//[^\n]*"))
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/frontend/parser.go b/frontend/parser.go
index bbbae8a..ac80bbe 100644
--- a/frontend/parser.go
+++ b/frontend/parser.go
@@ -18,22 +18,22 @@ type ParseError struct {
 	Reason     string
 	Production string
 	TC         int
-	text       []rune
+	text       []byte
 	chain      []*ParseError
 }
 
 // Errorf constructs a parse error with format for a particular location.
-func Errorf(text []rune, tc int, format string, args ...interface{}) *ParseError {
+func Errorf(text []byte, tc int, format string, args ...interface{}) *ParseError {
 	pc, _, _, ok := runtime.Caller(1)
 	return errorf(pc, ok, text, tc, format, args...)
 }
 
-func matchErrorf(text []rune, tc int, format string, args ...interface{}) *ParseError {
+func matchErrorf(text []byte, tc int, format string, args ...interface{}) *ParseError {
 	pc, _, _, ok := runtime.Caller(2)
 	return errorf(pc, ok, text, tc, format, args...)
 }
 
-func errorf(pc uintptr, ok bool, text []rune, tc int, format string, args ...interface{}) *ParseError {
+func errorf(pc uintptr, ok bool, text []byte, tc int, format string, args ...interface{}) *ParseError {
 	var fn = "unknown"
 	if ok {
 		fn = runtime.FuncForPC(pc).Name()
@@ -78,7 +78,7 @@ func (p *ParseError) Chain(e *ParseError) *ParseError {
 
 // LineCol computes the line and column of a particular index inside of a byte
 // slice.
-func LineCol(text []rune, tc int) (line int, col int) {
+func LineCol(text []byte, tc int) (line int, col int) {
 	for i := 0; i <= tc && i < len(text); i++ {
 		if text[i] == '\n' {
 			col = 0
@@ -97,7 +97,7 @@ func LineCol(text []rune, tc int) (line int, col int) {
 }
 
 // Parse a regular expression into an Abstract Syntax Tree (AST)
-func Parse(text []rune) (AST, error) {
+func Parse(text []byte) (AST, error) {
 	a, err := (&parser{
 		text:      text,
 		lastError: Errorf(text, 0, "unconsumed input"),
@@ -109,7 +109,7 @@ func Parse(text []rune) (AST, error) {
 }
 
 type parser struct {
-	text      []rune
+	text      []byte
 	lastError *ParseError
 }
 
@@ -349,7 +349,7 @@ func (p *parser) CHAR(i int) (int, AST, *ParseError) {
 	switch p.text[i] {
 	case '|', '+', '*', '?', '(', ')', '[', ']', '^':
 		return i, nil, Errorf(p.text, i,
-			"unexpected operator, %s", string([]rune{p.text[i]}))
+			"unexpected operator, %s", string([]byte{p.text[i]}))
 	case '.':
 		return i + 1, NewAny(), nil
 	default:
@@ -379,7 +379,7 @@ var (
 
 func (p *parser) builtInClass(i int) (int, AST, *ParseError) {
 	if p.text[i] != '\\' {
-		return i, nil, Errorf(p.text, i, "Not the start of built-in character class %q", string([]rune{p.text[i]}))
+		return i, nil, Errorf(p.text, i, "Not the start of built-in character class %q", string([]byte{p.text[i]}))
 	}
 	if i+1 < len(p.text) {
 		if p.text[i+1] == 'd' {
@@ -395,12 +395,12 @@ func (p *parser) builtInClass(i int) (int, AST, *ParseError) {
 		} else if p.text[i+1] == 'W' {
 			return i + 2, rangesToAST(builtInW), nil
 		}
-		return i, nil, Errorf(p.text, i, "Unknown class %q", string([]rune{p.text[i+1]}))
+		return i, nil, Errorf(p.text, i, "Unknown class %q", string([]byte{p.text[i+1]}))
 	}
 	return i, nil, Errorf(p.text, i, "Unexpected EOS")
 }
 
-func (p *parser) getByte(i int) (int, rune, *ParseError) {
+func (p *parser) getByte(i int) (int, byte, *ParseError) {
 	i, err := p.match(i, '\\')
 	if err == nil {
 		if i >= len(p.text) {
@@ -554,7 +554,7 @@ func (p *parser) matchAny(i int) (int, *Character, *ParseError) {
 	return i + 1, NewCharacter(p.text[i]), nil
 }
 
-func (p *parser) match(i int, c rune) (int, *ParseError) {
+func (p *parser) match(i int, c byte) (int, *ParseError) {
 	if i >= len(p.text) {
 		return i, matchErrorf(p.text, i, "out of p.text, %d", i)
 	} else if p.text[i] == c {
@@ -563,7 +563,7 @@ func (p *parser) match(i int, c rune) (int, *ParseError) {
 	}
 	return i, matchErrorf(p.text, i,
 		"expected '%v' at %v got '%v' of '%v'",
-		string([]rune{c}),
+		string([]byte{c}),
 		i,
 		string(p.text[i:i+1]),
 		string(p.text[i:]),
diff --git a/grammar b/grammar
new file mode 100644
index 0000000..b686b59
--- /dev/null
+++ b/grammar
@@ -0,0 +1,36 @@
+Regex : Alternation
+
+Alternation : AtomicOps Alternation'
+
+Alternation' : '|' AtomicOps Alternation'
+             | e
+
+AtomicOps : AtomicOp AtomicOps
+          | e
+
+AtomicOp : Atomic
+         | Atomic Op
+
+Atomic : Concat
+       | Group
+
+Op : '+'
+   | '*'
+   | '?'
+
+Group : '(' Alternation ')'
+
+Concat : Char Concat
+       | e
+
+Char : CHAR
+     | CharRange
+
+CharRange : '[' RangeInner ']'
+          | '[' '^' RangeInner ']'
+
+RangeInner : RangeComponent RangeInner
+           | e
+
+RangeComponent : CHAR
+               | CHAR '-' CHAR
diff --git a/lexc/main.go b/lexc/main.go
index 1606ff4..9e933c8 100644
--- a/lexc/main.go
+++ b/lexc/main.go
@@ -70,7 +70,7 @@ func main() {
 
 	asts := make([]frontend.AST, 0, len(patterns))
 	for _, p := range patterns {
-		ast, err := frontend.Parse([]rune(p))
+		ast, err := frontend.Parse([]byte(p))
 		if err != nil {
 			log.Fatal(err)
 		}
diff --git a/lexer.go b/lexer.go
index 6765c72..73241d7 100644
--- a/lexer.go
+++ b/lexer.go
@@ -1,6 +1,7 @@
 package lexmachine
 
 import (
+	"bytes"
 	"fmt"
 
 	dfapkg "gitea.xintech.co/zhouzhihong/lexmachine/dfa"
@@ -23,7 +24,7 @@ import (
 type Token struct {
 	Type        int
 	Value       interface{}
-	Lexeme      []rune
+	Lexeme      []byte
 	TC          int
 	StartLine   int
 	StartColumn int
@@ -45,7 +46,7 @@ func (t *Token) Equals(other *Token) bool {
 		t.StartColumn == other.StartColumn &&
 		t.EndLine == other.EndLine &&
 		t.EndColumn == other.EndColumn &&
-		string(t.Lexeme) == string(other.Lexeme) &&
+		bytes.Equal(t.Lexeme, other.Lexeme) &&
 		t.Type == other.Type
 }
 
@@ -63,7 +64,7 @@ func (t *Token) String() string {
 type Action func(scan *Scanner, match *machines.Match) (interface{}, error)
 
 type pattern struct {
-	regex  []rune
+	regex  []byte
 	action Action
 }
 
@@ -104,7 +105,7 @@ type Scanner struct {
 	lexer   *Lexer
 	matches map[int]int
 	scan    machines.Scanner
-	Text    []rune
+	Text    []byte
 	TC      int
 	pTC     int
 	sLine   int
@@ -183,7 +184,7 @@ func NewLexer() *Lexer {
 }
 
 // Scanner creates a scanner for a particular byte string from the lexer.
-func (l *Lexer) Scanner(text []rune) (*Scanner, error) {
+func (l *Lexer) Scanner(text []byte) (*Scanner, error) {
 	if l.program == nil && l.dfa == nil {
 		err := l.Compile()
 		if err != nil {
@@ -192,7 +193,7 @@ func (l *Lexer) Scanner(text []rune) (*Scanner, error) {
 	}
 
 	// prevent the user from modifying the text under scan
-	textCopy := make([]rune, len(text))
+	textCopy := make([]byte, len(text))
 	copy(textCopy, text)
 
 	var s *Scanner
@@ -219,7 +220,7 @@ func (l *Lexer) Scanner(text []rune) (*Scanner, error) {
 // Add pattern to match on. When a match occurs during scanning the action
 // function will be called by the Scanner to turn the low level machines.Match
 // struct into a token.
-func (l *Lexer) Add(regex []rune, action Action) {
+func (l *Lexer) Add(regex []byte, action Action) {
 	if l.program != nil {
 		l.program = nil
 	}
@@ -320,7 +321,7 @@ func (l *Lexer) CompileDFA() error {
 }
 
 func (l *Lexer) matchesEmptyString() (bool, error) {
-	s, err := l.Scanner([]rune(""))
+	s, err := l.Scanner([]byte(""))
 	if err != nil {
 		return false, err
 	}
diff --git a/lexer_test.go b/lexer_test.go
index f5e3842..a80815e 100644
--- a/lexer_test.go
+++ b/lexer_test.go
@@ -21,25 +21,25 @@ func TestSimple(x *testing.T) {
 	lexer := NewLexer()
 
 	lexer.Add(
-		[]rune("print"),
+		[]byte("print"),
 		func(scan *Scanner, match *machines.Match) (interface{}, error) {
 			return scan.Token(PRINT, nil, match), nil
 		},
 	)
 	lexer.Add(
-		[]rune("([a-z]|[A-Z])([a-z]|[A-Z]|[0-9]|_)*"),
+		[]byte("([a-z]|[A-Z])([a-z]|[A-Z]|[0-9]|_)*"),
 		func(scan *Scanner, match *machines.Match) (interface{}, error) {
 			return scan.Token(NAME, string(match.Bytes), match), nil
 		},
 	)
 	lexer.Add(
-		[]rune("="),
+		[]byte("="),
 		func(scan *Scanner, match *machines.Match) (interface{}, error) {
 			return scan.Token(EQUALS, nil, match), nil
 		},
 	)
 	lexer.Add(
-		[]rune("[0-9]+"),
+		[]byte("[0-9]+"),
 		func(scan *Scanner, match *machines.Match) (interface{}, error) {
 			i, err := strconv.Atoi(string(match.Bytes))
 			if err != nil {
@@ -49,21 +49,21 @@ func TestSimple(x *testing.T) {
 		},
 	)
 	lexer.Add(
-		[]rune("( |\t|\n)"),
+		[]byte("( |\t|\n)"),
 		func(scan *Scanner, match *machines.Match) (interface{}, error) {
 			// skip white space
 			return nil, nil
 		},
 	)
 	lexer.Add(
-		[]rune("//[^\n]*\n"),
+		[]byte("//[^\n]*\n"),
 		func(scan *Scanner, match *machines.Match) (interface{}, error) {
 			// skip white space
 			return nil, nil
 		},
 	)
 	lexer.Add(
-		[]rune("/\\*"),
+		[]byte("/\\*"),
 		func(scan *Scanner, match *machines.Match) (interface{}, error) {
 			for tc := scan.TC; tc < len(scan.Text); tc++ {
 				if scan.Text[tc] == '\\' {
@@ -82,7 +82,7 @@ func TestSimple(x *testing.T) {
 		},
 	)
 
-	text := []rune(`
+	text := []byte(`
 		name = 10
 		print name
 		print fred
@@ -95,21 +95,21 @@ func TestSimple(x *testing.T) {
 	`)
 
 	expected := []*Token{
-		{NAME, "name", []rune("name"), 3, 2, 3, 2, 6},
-		{EQUALS, nil, []rune("="), 8, 2, 8, 2, 8},
-		{NUMBER, 10, []rune("10"), 10, 2, 10, 2, 11},
-		{PRINT, nil, []rune("print"), 15, 3, 3, 3, 7},
-		{NAME, "name", []rune("name"), 21, 3, 9, 3, 12},
-		{PRINT, nil, []rune("print"), 28, 4, 3, 4, 7},
-		{NAME, "fred", []rune("fred"), 34, 4, 9, 4, 12},
-		{NAME, "name", []rune("name"), 41, 5, 3, 5, 6},
-		{EQUALS, nil, []rune("="), 46, 5, 8, 5, 8},
-		{NUMBER, 12, []rune("12"), 47, 5, 9, 5, 10},
-		{NAME, "printname", []rune("printname"), 112, 9, 11, 9, 19},
-		{EQUALS, nil, []rune("="), 122, 9, 21, 9, 21},
-		{NUMBER, 13, []rune("13"), 124, 9, 23, 9, 24},
-		{PRINT, nil, []rune("print"), 129, 10, 3, 10, 7},
-		{NAME, "printname", []rune("printname"), 135, 10, 9, 10, 17},
+		{NAME, "name", []byte("name"), 3, 2, 3, 2, 6},
+		{EQUALS, nil, []byte("="), 8, 2, 8, 2, 8},
+		{NUMBER, 10, []byte("10"), 10, 2, 10, 2, 11},
+		{PRINT, nil, []byte("print"), 15, 3, 3, 3, 7},
+		{NAME, "name", []byte("name"), 21, 3, 9, 3, 12},
+		{PRINT, nil, []byte("print"), 28, 4, 3, 4, 7},
+		{NAME, "fred", []byte("fred"), 34, 4, 9, 4, 12},
+		{NAME, "name", []byte("name"), 41, 5, 3, 5, 6},
+		{EQUALS, nil, []byte("="), 46, 5, 8, 5, 8},
+		{NUMBER, 12, []byte("12"), 47, 5, 9, 5, 10},
+		{NAME, "printname", []byte("printname"), 112, 9, 11, 9, 19},
+		{EQUALS, nil, []byte("="), 122, 9, 21, 9, 21},
+		{NUMBER, 13, []byte("13"), 124, 9, 23, 9, 24},
+		{PRINT, nil, []byte("print"), 129, 10, 3, 10, 7},
+		{NAME, "printname", []byte("printname"), 135, 10, 9, 10, 17},
 	}
 
 	scan := func(lexer *Lexer) {
@@ -221,15 +221,15 @@ func TestPartialLexer(x *testing.T) {
 		}
 	}
 	var lexer = NewLexer()
-	lexer.Add([]rune("import|require"), getToken(tokmap["INCLUDE"]))
-	lexer.Add([]rune("function"), getToken(tokmap["FUNC"]))
-	lexer.Add([]rune("class"), getToken(tokmap["CLASS"]))
-	lexer.Add([]rune("\"[^\\\"]*\"|'[^']*'|`[^`]*`"), getToken(tokmap["STRING"]))
-	lexer.Add([]rune("//[^\n]*\n?|/\\*([^*]|\r|\n|(\\*+([^*/]|\r|\n)))*\\*+/"), getToken(tokmap["COMMENT"]))
-	lexer.Add([]rune("[A-Za-z$][A-Za-z0-9$]+"), getToken(tokmap["IDENT"]))
-	lexer.Add([]rune(">=|<=|=|>|<|\\|\\||&&"), getToken(tokmap["OP"]))
+	lexer.Add([]byte("import|require"), getToken(tokmap["INCLUDE"]))
+	lexer.Add([]byte("function"), getToken(tokmap["FUNC"]))
+	lexer.Add([]byte("class"), getToken(tokmap["CLASS"]))
+	lexer.Add([]byte("\"[^\\\"]*\"|'[^']*'|`[^`]*`"), getToken(tokmap["STRING"]))
+	lexer.Add([]byte("//[^\n]*\n?|/\\*([^*]|\r|\n|(\\*+([^*/]|\r|\n)))*\\*+/"), getToken(tokmap["COMMENT"]))
+	lexer.Add([]byte("[A-Za-z$][A-Za-z0-9$]+"), getToken(tokmap["IDENT"]))
+	lexer.Add([]byte(">=|<=|=|>|<|\\|\\||&&"), getToken(tokmap["OP"]))
 	scan := func(lexer *Lexer) {
-		scanner, err := lexer.Scanner([]rune(text))
+		scanner, err := lexer.Scanner([]byte(text))
 		t.AssertNil(err)
 		i := 0
 		for tk, err, eof := scanner.Next(); !eof; tk, err, eof = scanner.Next() {
@@ -263,8 +263,8 @@ func TestRegression(t *testing.T) {
 
 	newLexer := func() *Lexer {
 		lexer := NewLexer()
-		lexer.Add([]rune("true"), token("TRUE"))
-		lexer.Add([]rune("( |\t|\n|\r)+"), token("SPACE"))
+		lexer.Add([]byte("true"), token("TRUE"))
+		lexer.Add([]byte("( |\t|\n|\r)+"), token("SPACE"))
 		return lexer
 	}
 
@@ -278,7 +278,7 @@ func TestRegression(t *testing.T) {
 
 	runTest := func(lexer *Lexer) {
 		for _, test := range tests {
-			scanner, err := lexer.Scanner([]rune(test.text))
+			scanner, err := lexer.Scanner([]byte(test.text))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -321,7 +321,7 @@ func TestRegression2(t *testing.T) {
 #
 
 # option definitions common to all supported networks...
-option domain-name "example.org";
+option domain-name "你好";
 option domain-name-servers ns1.example.org, ns2.example.org;
 
 default-lease-time 600;
@@ -365,18 +365,18 @@ ddns-update-style none;
 			}
 		}
 
-		lex.Add([]rune(`#[^\n]*\n?`), token("COMMENT"))
-		lex.Add([]rune(`([a-z]|[A-Z]|[0-9]|_|\-|\.)+`), token("ID"))
-		lex.Add([]rune(`"([^\\"]|(\\.))*"`), token("ID"))
-		lex.Add([]rune("[\n \t]"), skip)
+		lex.Add([]byte(`#[^\n]*\n?`), token("COMMENT"))
+		lex.Add([]byte(`([a-z]|[A-Z]|[0-9]|_|\-|\.)+`), token("ID"))
+		lex.Add([]byte(`"([^\\"]|(\\.))*"`), token("ID"))
+		lex.Add([]byte("[\n \t]"), skip)
 		for _, lit := range literals {
-			lex.Add([]rune(lit), token(lit))
+			lex.Add([]byte(lit), token(lit))
 		}
 		return lex
 	}
 
 	runTest := func(lexer *Lexer) {
-		scanner, err := lexer.Scanner([]rune(text))
+		scanner, err := lexer.Scanner([]byte(text))
 		if err != nil {
 			return
 		}
@@ -436,12 +436,12 @@ func TestPythonStrings(t *testing.T) {
 
 	newLexer := func() *Lexer {
 		lexer := NewLexer()
-		lexer.Add([]rune("true"), token("TRUE"))
-		lexer.Add([]rune(`'''([^\\']|(\\.))*'''`), token("TRIPLE_STRING"))
-		lexer.Add([]rune(`"""([^\\"]|(\\.))*"""`), token("TRIPLE_STRING"))
-		lexer.Add([]rune(`"([^\\"]|(\\.))*"`), token("SINGLE_STRING"))
-		lexer.Add([]rune(`'([^\\']|(\\.))*'`), token("SINGLE_STRING"))
-		lexer.Add([]rune("( |\t|\n|\r)+"), skip)
+		lexer.Add([]byte("true"), token("TRUE"))
+		lexer.Add([]byte(`'''([^\\']|(\\.))*'''`), token("TRIPLE_STRING"))
+		lexer.Add([]byte(`"""([^\\"]|(\\.))*"""`), token("TRIPLE_STRING"))
+		lexer.Add([]byte(`"([^\\"]|(\\.))*"`), token("SINGLE_STRING"))
+		lexer.Add([]byte(`'([^\\']|(\\.))*'`), token("SINGLE_STRING"))
+		lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 		return lexer
 	}
 
@@ -468,7 +468,7 @@ func TestPythonStrings(t *testing.T) {
 	runTest := func(lexer *Lexer) {
 		for _, test := range tests {
 			fmt.Printf("test %q\n", test.text)
-			scanner, err := lexer.Scanner([]rune(test.text))
+			scanner, err := lexer.Scanner([]byte(test.text))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -520,7 +520,7 @@ func TestNoEmptyStrings(t *testing.T) {
 		return nil, nil
 	}
 	lexer := NewLexer()
-	lexer.Add([]rune("(ab|a)*"), skip)
+	lexer.Add([]byte("(ab|a)*"), skip)
 	{
 		if err := lexer.CompileNFA(); err == nil {
 			t.Fatal("expected error")
diff --git a/machines/dfa_machine.go b/machines/dfa_machine.go
index 04ef241..e065f4c 100644
--- a/machines/dfa_machine.go
+++ b/machines/dfa_machine.go
@@ -13,7 +13,8 @@ type lineCol struct {
 }
 
 // Compute the line and column of a particular index inside of a byte slice.
-func mapLineCols(text []rune) []lineCol {
+func mapLineCols(text []byte) []lineCol {
+	// text := []rune(string(t))
 	m := make([]lineCol, len(text))
 	line := 1
 	col := 0
@@ -32,7 +33,7 @@ func mapLineCols(text []rune) []lineCol {
 // DFALexerEngine does the actual tokenization of the byte slice text using the
 // DFA state machine. If the lexing process fails the Scanner will return
 // an UnconsumedInput error.
-func DFALexerEngine(startState, errorState int, trans DFATrans, accepting DFAAccepting, text []rune) Scanner {
+func DFALexerEngine(startState, errorState int, trans DFATrans, accepting DFAAccepting, text []byte) Scanner {
 	lineCols := mapLineCols(text)
 	done := false
 	matchID := -1
diff --git a/machines/machine.go b/machines/machine.go
index b3af1d3..e60ccff 100644
--- a/machines/machine.go
+++ b/machines/machine.go
@@ -2,6 +2,7 @@
 package machines
 
 import (
+	"bytes"
 	"fmt"
 
 	"gitea.xintech.co/zhouzhihong/lexmachine/inst"
@@ -31,7 +32,7 @@ type UnconsumedInput struct {
 	StartColumn int
 	FailLine    int
 	FailColumn  int
-	Text        []rune
+	Text        []byte
 }
 
 // Error implements the error interface
@@ -65,38 +66,39 @@ type Match struct {
 	StartColumn int
 	EndLine     int
 	EndColumn   int
-	Bytes       []rune // the actual bytes matched during scanning.
+	Bytes       []byte // the actual bytes matched during scanning.
 }
 
-func computeLineCol(text []rune, prevTC, tc, line, col int) (int, int) {
+func computeLineCol(s []byte, prevTC, tc, line, col int) (int, int) {
+	// s := []rune(string(text))
 	if tc < 0 {
 		return line, col
 	}
 	if tc < prevTC {
 		for i := prevTC; i > tc && i > 0; i-- {
-			if text[i] == '\n' {
+			if s[i] == '\n' {
 				line--
 			}
 		}
 		col = 0
 		for i := tc; i >= 0; i-- {
-			if text[i] == '\n' {
+			if s[i] == '\n' {
 				break
 			}
 			col++
 		}
 		return line, col
 	}
-	for i := prevTC + 1; i <= tc && i < len(text); i++ {
-		if text[i] == '\n' {
+	for i := prevTC + 1; i <= tc && i < len(s); i++ {
+		if s[i] == '\n' {
 			col = 0
 			line++
 		} else {
 			col++
 		}
 	}
-	if prevTC == tc && tc == 0 && tc < len(text) {
-		if text[tc] == '\n' {
+	if prevTC == tc && tc == 0 && tc < len(s) {
+		if s[tc] == '\n' {
 			line++
 			col--
 		}
@@ -118,7 +120,7 @@ func (m *Match) Equals(other *Match) bool {
 		m.StartColumn == other.StartColumn &&
 		m.EndLine == other.EndLine &&
 		m.EndColumn == other.EndColumn &&
-		string(m.Bytes) == string(other.Bytes)
+		bytes.Equal(m.Bytes, other.Bytes)
 }
 
 // String formats the match for humans
@@ -133,7 +135,7 @@ type Scanner func(int) (int, *Match, error, Scanner)
 // LexerEngine does the actual tokenization of the byte slice text using the
 // NFA bytecode in program. If the lexing process fails the Scanner will return
 // an UnconsumedInput error.
-func LexerEngine(program inst.Slice, text []rune) Scanner {
+func LexerEngine(program inst.Slice, text []byte) Scanner {
 	done := false
 	matchPC := -1
 	matchTC := -1
@@ -170,8 +172,8 @@ func LexerEngine(program inst.Slice, text []rune) Scanner {
 				i := program[pc]
 				switch i.Op {
 				case inst.CHAR:
-					x := rune(i.X)
-					y := rune(i.Y)
+					x := byte(i.X)
+					y := byte(i.Y)
 					if tc < len(text) && x <= text[tc] && text[tc] <= y {
 						nqueue.Push(pc + 1)
 					}
diff --git a/machines/machine_test.go b/machines/machine_test.go
index 0f4ee65..1b42d05 100644
--- a/machines/machine_test.go
+++ b/machines/machine_test.go
@@ -7,7 +7,7 @@ import (
 )
 
 func TestLexerMatch(t *testing.T) {
-	text := []rune("ababcbcbb")
+	text := []byte("ababcbcbb")
 	//. (a|b)*cba?(c|b)bb
 	program := make(inst.Slice, 20)
 
@@ -32,7 +32,7 @@ func TestLexerMatch(t *testing.T) {
 	t.Log(string(text))
 	t.Log(len(text))
 	t.Log(program)
-	mtext := []rune("ababcbcbb")
+	mtext := []byte("ababcbcbb")
 	expected := []Match{
 		{16, 0, 1, 1, 1, len(mtext), mtext},
 	}
@@ -52,7 +52,7 @@ func TestLexerMatch(t *testing.T) {
 }
 
 func TestLexerNoMatch(t *testing.T) {
-	text := []rune("ababcbcb")
+	text := []byte("ababcbcb")
 	//. (a|b)*cba?(c|b)bb
 	program := make(inst.Slice, 20)
 
@@ -90,7 +90,7 @@ func TestLexerNoMatch(t *testing.T) {
 }
 
 func TestLexerThreeStrings(t *testing.T) {
-	var text = []rune("struct  *")
+	var text = []byte("struct  *")
 	program := make(inst.Slice, 30)
 
 	program[0] = inst.New(inst.SPLIT, 2, 1)  // go to 1 or 2/3
@@ -114,9 +114,9 @@ func TestLexerThreeStrings(t *testing.T) {
 	t.Log(len(text))
 	t.Log(program)
 	expected := []Match{
-		{8, 0, 1, 1, 1, 6, []rune("struct")},
-		{13, 6, 1, 7, 1, 8, []rune("  ")},
-		{15, 8, 1, 9, 1, 9, []rune("*")},
+		{8, 0, 1, 1, 1, 6, []byte("struct")},
+		{13, 6, 1, 7, 1, 8, []byte("  ")},
+		{15, 8, 1, 9, 1, 9, []byte("*")},
 	}
 
 	i := 0
@@ -135,7 +135,7 @@ func TestLexerThreeStrings(t *testing.T) {
 }
 
 func TestLexerRestart(t *testing.T) {
-	var text = []rune("struct\n  *")
+	var text = []byte("struct\n  *")
 	program := make(inst.Slice, 30)
 
 	program[0] = inst.New(inst.SPLIT, 2, 1)  // go to 1 or 2/3
@@ -165,9 +165,9 @@ func TestLexerRestart(t *testing.T) {
 	t.Log(len(text))
 	t.Log(program)
 	expected := []Match{
-		{8, 0, 1, 1, 1, 6, []rune("struct")},
-		{19, 6, 2, 0, 2, 2, []rune("\n  ")},
-		{21, 9, 2, 3, 2, 3, []rune("*")},
+		{8, 0, 1, 1, 1, 6, []byte("struct")},
+		{19, 6, 2, 0, 2, 2, []byte("\n  ")},
+		{21, 9, 2, 3, 2, 3, []byte("*")},
 	}
 
 	check := func(m *Match, i int, err error) {
-- 
2.45.2


From 2da032b556e10008a0475a46b9a55a0c26a8a1e6 Mon Sep 17 00:00:00 2001
From: zhouzhihong <zhouzhihong@ijunhai.com>
Date: Thu, 25 Aug 2022 10:49:30 +0800
Subject: [PATCH 2/3] Revert code

---
 cmd/main.go         | 23 +++++++++++++++++++++++
 machines/machine.go | 15 +++++++--------
 2 files changed, 30 insertions(+), 8 deletions(-)
 create mode 100644 cmd/main.go

diff --git a/cmd/main.go b/cmd/main.go
new file mode 100644
index 0000000..eb958b6
--- /dev/null
+++ b/cmd/main.go
@@ -0,0 +1,23 @@
+package main
+
+import (
+	"fmt"
+	"unicode/utf8"
+)
+
+func main() {
+	b := []byte("你好")
+
+	sz := 0
+	var r rune
+
+	for i := 0; i < len(b); {
+		r, sz = utf8.DecodeRune(b[i:])
+		i += sz
+
+		fmt.Println(r, sz, i, string(r))
+	}
+
+	// fmt.Println(len(b))
+
+}
diff --git a/machines/machine.go b/machines/machine.go
index e60ccff..b6325b0 100644
--- a/machines/machine.go
+++ b/machines/machine.go
@@ -69,36 +69,35 @@ type Match struct {
 	Bytes       []byte // the actual bytes matched during scanning.
 }
 
-func computeLineCol(s []byte, prevTC, tc, line, col int) (int, int) {
-	// s := []rune(string(text))
+func computeLineCol(text []byte, prevTC, tc, line, col int) (int, int) {
 	if tc < 0 {
 		return line, col
 	}
 	if tc < prevTC {
 		for i := prevTC; i > tc && i > 0; i-- {
-			if s[i] == '\n' {
+			if text[i] == '\n' {
 				line--
 			}
 		}
 		col = 0
 		for i := tc; i >= 0; i-- {
-			if s[i] == '\n' {
+			if text[i] == '\n' {
 				break
 			}
 			col++
 		}
 		return line, col
 	}
-	for i := prevTC + 1; i <= tc && i < len(s); i++ {
-		if s[i] == '\n' {
+	for i := prevTC + 1; i <= tc && i < len(text); i++ {
+		if text[i] == '\n' {
 			col = 0
 			line++
 		} else {
 			col++
 		}
 	}
-	if prevTC == tc && tc == 0 && tc < len(s) {
-		if s[tc] == '\n' {
+	if prevTC == tc && tc == 0 && tc < len(text) {
+		if text[tc] == '\n' {
 			line++
 			col--
 		}
-- 
2.45.2


From 3d56a5d5f1c9a802938f5f8b564ea75e577a6f1e Mon Sep 17 00:00:00 2001
From: zhouzhihong <zhouzhihong@ijunhai.com>
Date: Thu, 25 Aug 2022 14:33:22 +0800
Subject: [PATCH 3/3] Add text line and text column calculator.

---
 frontend/frontend_test.go |  2 +-
 lexer.go                  | 48 +++++++++++++++++++++++++++++++++++++++
 lexer_test.go             | 21 +----------------
 machines/machine.go       |  2 ++
 machines/machine_test.go  | 14 ++++++------
 5 files changed, 59 insertions(+), 28 deletions(-)

diff --git a/frontend/frontend_test.go b/frontend/frontend_test.go
index de345c9..22f9c07 100644
--- a/frontend/frontend_test.go
+++ b/frontend/frontend_test.go
@@ -23,7 +23,7 @@ func TestParse(x *testing.T) {
 }
 
 func tMatch(program inst.Slice, text string, t *test.T) {
-	expected := []machines.Match{{len(program) - 1, 0, 1, 1, 1, len(text), []byte(text)}}
+	expected := []machines.Match{{PC: len(program) - 1, TC: 0, StartLine: 1, StartColumn: 1, EndLine: 1, EndColumn: len(text), Bytes: []byte(text), TSLine: 1, TSColumn: 0, TELine: 1, TEColumn: 1}}
 	if expected[0].EndColumn == 0 {
 		expected[0].EndColumn = 1
 	}
diff --git a/lexer.go b/lexer.go
index 73241d7..ea6d37e 100644
--- a/lexer.go
+++ b/lexer.go
@@ -3,6 +3,7 @@ package lexmachine
 import (
 	"bytes"
 	"fmt"
+	"unicode/utf8"
 
 	dfapkg "gitea.xintech.co/zhouzhihong/lexmachine/dfa"
 	"gitea.xintech.co/zhouzhihong/lexmachine/frontend"
@@ -112,6 +113,12 @@ type Scanner struct {
 	sColumn int
 	eLine   int
 	eColumn int
+
+	lpp map[int]lastPostion
+}
+
+type lastPostion struct {
+	l, c int
 }
 
 // Next iterates through the string being scanned returning one token at a time
@@ -155,6 +162,38 @@ func (s *Scanner) Next() (tok interface{}, err error, eos bool) {
 		s.eLine = match.EndLine
 		s.eColumn = match.EndColumn
 
+		p := s.pTC
+		l, c := s.lpp[p].l, s.lpp[p].c
+		stc := s.TC - len(match.Bytes)
+
+		for {
+			if s.Text[p] == '\n' {
+				l++
+				c = 0
+			} else {
+				c++
+			}
+
+			if p == stc {
+				match.TSLine = l
+				match.TSColumn = c
+			}
+
+			match.TELine = l
+			match.TEColumn = c
+
+			_, sz := utf8.DecodeRune(s.Text[p:])
+			p += sz
+			if p >= s.TC {
+				break
+			}
+		}
+
+		s.lpp[s.TC] = lastPostion{
+			l: l,
+			c: c,
+		}
+
 		pattern := s.lexer.patterns[s.matches[match.PC]]
 		token, err = pattern.action(s, match)
 		if err != nil {
@@ -204,6 +243,7 @@ func (l *Lexer) Scanner(text []byte) (*Scanner, error) {
 			scan:    machines.DFALexerEngine(l.dfa.Start, l.dfa.Error, l.dfa.Trans, l.dfa.Accepting, textCopy),
 			Text:    textCopy,
 			TC:      0,
+			lpp:     make(map[int]lastPostion),
 		}
 	} else {
 		s = &Scanner{
@@ -212,8 +252,16 @@ func (l *Lexer) Scanner(text []byte) (*Scanner, error) {
 			scan:    machines.LexerEngine(l.program, textCopy),
 			Text:    textCopy,
 			TC:      0,
+			lpp:     make(map[int]lastPostion),
 		}
 	}
+
+	//init
+	s.lpp[0] = lastPostion{
+		l: 1,
+		c: 0,
+	}
+
 	return s, nil
 }
 
diff --git a/lexer_test.go b/lexer_test.go
index a80815e..79e9797 100644
--- a/lexer_test.go
+++ b/lexer_test.go
@@ -316,26 +316,7 @@ func TestRegression(t *testing.T) {
 func TestRegression2(t *testing.T) {
 
 	text := `# dhcpd.conf
-#
-# Sample configuration file for ISC dhcpd
-#
-
-# option definitions common to all supported networks...
-option domain-name "你好";
-option domain-name-servers ns1.example.org, ns2.example.org;
-
-default-lease-time 600;
-max-lease-time 7200;
-
-# The ddns-updates-style parameter controls whether or not the server will
-# attempt to do a DNS update when a lease is confirmed. We default to the
-# behavior of the version 2 packages ('none', since DHCP v2 didn't
-# have support for DDNS.)
-ddns-update-style none;
-
-# If this DHCP server is the official DHCP server for the local
-# network, the authoritative directive should be uncommented.
-#authoritative;
+option domain-name "你好"
 `
 
 	literals := []string{
diff --git a/machines/machine.go b/machines/machine.go
index b6325b0..e4c44e0 100644
--- a/machines/machine.go
+++ b/machines/machine.go
@@ -67,6 +67,8 @@ type Match struct {
 	EndLine     int
 	EndColumn   int
 	Bytes       []byte // the actual bytes matched during scanning.
+
+	TSLine, TSColumn, TELine, TEColumn int
 }
 
 func computeLineCol(text []byte, prevTC, tc, line, col int) (int, int) {
diff --git a/machines/machine_test.go b/machines/machine_test.go
index 1b42d05..b0ad37e 100644
--- a/machines/machine_test.go
+++ b/machines/machine_test.go
@@ -34,7 +34,7 @@ func TestLexerMatch(t *testing.T) {
 	t.Log(program)
 	mtext := []byte("ababcbcbb")
 	expected := []Match{
-		{16, 0, 1, 1, 1, len(mtext), mtext},
+		{16, 0, 1, 1, 1, len(mtext), mtext, 1, 0, 1, 1},
 	}
 	i := 0
 	for tc, m, err, scan := LexerEngine(program, text)(0); scan != nil; tc, m, err, scan = scan(tc) {
@@ -114,9 +114,9 @@ func TestLexerThreeStrings(t *testing.T) {
 	t.Log(len(text))
 	t.Log(program)
 	expected := []Match{
-		{8, 0, 1, 1, 1, 6, []byte("struct")},
-		{13, 6, 1, 7, 1, 8, []byte("  ")},
-		{15, 8, 1, 9, 1, 9, []byte("*")},
+		{8, 0, 1, 1, 1, 6, []byte("struct"), 1, 0, 1, 1},
+		{13, 6, 1, 7, 1, 8, []byte("  "), 1, 0, 1, 1},
+		{15, 8, 1, 9, 1, 9, []byte("*"), 1, 0, 1, 1},
 	}
 
 	i := 0
@@ -165,9 +165,9 @@ func TestLexerRestart(t *testing.T) {
 	t.Log(len(text))
 	t.Log(program)
 	expected := []Match{
-		{8, 0, 1, 1, 1, 6, []byte("struct")},
-		{19, 6, 2, 0, 2, 2, []byte("\n  ")},
-		{21, 9, 2, 3, 2, 3, []byte("*")},
+		{8, 0, 1, 1, 1, 6, []byte("struct"), 1, 0, 1, 1},
+		{19, 6, 2, 0, 2, 2, []byte("\n  "), 1, 0, 1, 1},
+		{21, 9, 2, 3, 2, 3, []byte("*"), 1, 0, 1, 1},
 	}
 
 	check := func(m *Match, i int, err error) {
-- 
2.45.2