Merge branch 'dev'

lexer.go

@@ -31,6 +31,8 @@ type Token struct {
 	StartColumn int
 	EndLine     int
 	EndColumn   int
+
+	TSLine, TSColumn, TELine, TEColumn int
 }
 
 // Equals checks the equality of two tokens ignoring the Value field.
@@ -152,7 +154,7 @@ func (s *Scanner) Next() (tok interface{}, err error, eos bool) {
 		} else if err != nil {
 			return nil, err, false
 		} else if match == nil {
-			return nil, fmt.Errorf("No match but no error"), false
+			return nil, fmt.Errorf("no match but no error"), false
 		}
 		s.scan = scan
 		s.pTC = s.TC
@@ -214,6 +216,10 @@ func (s *Scanner) Token(typ int, value interface{}, m *machines.Match) *Token {
 		StartColumn: m.StartColumn,
 		EndLine:     m.EndLine,
 		EndColumn:   m.EndColumn,
+		TSLine:      m.TSLine,
+		TSColumn:    m.TSColumn,
+		TELine:      m.TELine,
+		TEColumn:    m.TEColumn,
 	}
 }
 
@@ -303,7 +309,7 @@ func (l *Lexer) assembleAST() (frontend.AST, error) {
 // only created explicitly) this will be used by Scanners when they are created.
 func (l *Lexer) CompileNFA() error {
 	if len(l.patterns) == 0 {
-		return fmt.Errorf("No patterns added")
+		return fmt.Errorf("no patterns added")
 	}
 	if l.program != nil {
 		return nil
@@ -333,7 +339,7 @@ func (l *Lexer) CompileNFA() error {
 	} else if mes {
 		l.program = nil
 		l.nfaMatches = nil
-		return fmt.Errorf("One or more of the supplied patterns match the empty string")
+		return fmt.Errorf("one or more of the supplied patterns match the empty string")
 	}
 
 	return nil
@@ -343,7 +349,7 @@ func (l *Lexer) CompileNFA() error {
 // they are created.
 func (l *Lexer) CompileDFA() error {
 	if len(l.patterns) == 0 {
-		return fmt.Errorf("No patterns added")
+		return fmt.Errorf("no patterns added")
 	}
 	if l.dfa != nil {
 		return nil
@@ -363,7 +369,7 @@ func (l *Lexer) CompileDFA() error {
 	} else if mes {
 		l.dfa = nil
 		l.dfaMatches = nil
-		return fmt.Errorf("One or more of the supplied patterns match the empty string")
+		return fmt.Errorf("one or more of the supplied patterns match the empty string")
 	}
 	return nil
 }

lexer_test.go

@@ -95,21 +95,21 @@ func TestSimple(x *testing.T) {
 	`)
 
 	expected := []*Token{
-		{NAME, "name", []byte("name"), 3, 2, 3, 2, 6},
+		{NAME, "name", []byte("name"), 3, 2, 3, 2, 6, 2, 3, 2, 6},
-		{EQUALS, nil, []byte("="), 8, 2, 8, 2, 8},
+		{EQUALS, nil, []byte("="), 8, 2, 8, 2, 8, 2, 8, 2, 8},
-		{NUMBER, 10, []byte("10"), 10, 2, 10, 2, 11},
+		{NUMBER, 10, []byte("10"), 10, 2, 10, 2, 11, 2, 10, 2, 11},
-		{PRINT, nil, []byte("print"), 15, 3, 3, 3, 7},
+		{PRINT, nil, []byte("print"), 15, 3, 3, 3, 7, 3, 3, 3, 7},
-		{NAME, "name", []byte("name"), 21, 3, 9, 3, 12},
+		{NAME, "name", []byte("name"), 21, 3, 9, 3, 12, 3, 9, 3, 12},
-		{PRINT, nil, []byte("print"), 28, 4, 3, 4, 7},
+		{PRINT, nil, []byte("print"), 28, 4, 3, 4, 7, 4, 3, 4, 7},
-		{NAME, "fred", []byte("fred"), 34, 4, 9, 4, 12},
+		{NAME, "fred", []byte("fred"), 34, 4, 9, 4, 12, 4, 9, 4, 12},
-		{NAME, "name", []byte("name"), 41, 5, 3, 5, 6},
+		{NAME, "name", []byte("name"), 41, 5, 3, 5, 6, 5, 3, 5, 6},
-		{EQUALS, nil, []byte("="), 46, 5, 8, 5, 8},
+		{EQUALS, nil, []byte("="), 46, 5, 8, 5, 8, 5, 8, 5, 8},
-		{NUMBER, 12, []byte("12"), 47, 5, 9, 5, 10},
+		{NUMBER, 12, []byte("12"), 47, 5, 9, 5, 10, 5, 9, 5, 10},
-		{NAME, "printname", []byte("printname"), 112, 9, 11, 9, 19},
+		{NAME, "printname", []byte("printname"), 112, 9, 11, 9, 19, 9, 11, 9, 19},
-		{EQUALS, nil, []byte("="), 122, 9, 21, 9, 21},
+		{EQUALS, nil, []byte("="), 122, 9, 21, 9, 21, 9, 21, 9, 21},
-		{NUMBER, 13, []byte("13"), 124, 9, 23, 9, 24},
+		{NUMBER, 13, []byte("13"), 124, 9, 23, 9, 24, 9, 23, 9, 24},
-		{PRINT, nil, []byte("print"), 129, 10, 3, 10, 7},
+		{PRINT, nil, []byte("print"), 129, 10, 3, 10, 7, 10, 3, 10, 7},
-		{NAME, "printname", []byte("printname"), 135, 10, 9, 10, 17},
+		{NAME, "printname", []byte("printname"), 135, 10, 9, 10, 17, 10, 9, 10, 17},
 	}
 
 	scan := func(lexer *Lexer) {