zhouzhihong/lexmachine
1
0
Fork 0
Code Issues Pull Requests Projects Releases 2 Wiki Activity

Merge branch 'dev'

Browse Source
This commit is contained in:
zhouzhihong 2022-08-25 15:18:09 +08:00
commit 48e1eef593
2 changed files with 26 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lexer.go
View File

@ -31,6 +31,8 @@ type Token struct {
StartColumn int StartColumn int
EndLine int EndLine int
EndColumn int EndColumn int
TSLine, TSColumn, TELine, TEColumn int
} }
// Equals checks the equality of two tokens ignoring the Value field. // Equals checks the equality of two tokens ignoring the Value field.
@ -152,7 +154,7 @@ func (s *Scanner) Next() (tok interface{}, err error, eos bool) {
} else if err != nil { } else if err != nil {
return nil, err, false return nil, err, false
} else if match == nil { } else if match == nil {
return nil, fmt.Errorf("No match but no error"), false return nil, fmt.Errorf("no match but no error"), false
} }
s.scan = scan s.scan = scan
s.pTC = s.TC s.pTC = s.TC
@ -214,6 +216,10 @@ func (s *Scanner) Token(typ int, value interface{}, m *machines.Match) *Token {
StartColumn: m.StartColumn, StartColumn: m.StartColumn,
EndLine: m.EndLine, EndLine: m.EndLine,
EndColumn: m.EndColumn, EndColumn: m.EndColumn,
TSLine: m.TSLine,
TSColumn: m.TSColumn,
TELine: m.TELine,
TEColumn: m.TEColumn,
} }
} }
@ -303,7 +309,7 @@ func (l *Lexer) assembleAST() (frontend.AST, error) {
// only created explicitly) this will be used by Scanners when they are created. // only created explicitly) this will be used by Scanners when they are created.
func (l *Lexer) CompileNFA() error { func (l *Lexer) CompileNFA() error {
if len(l.patterns) == 0 { if len(l.patterns) == 0 {
return fmt.Errorf("No patterns added") return fmt.Errorf("no patterns added")
} }
if l.program != nil { if l.program != nil {
return nil return nil
@ -333,7 +339,7 @@ func (l *Lexer) CompileNFA() error {
} else if mes { } else if mes {
l.program = nil l.program = nil
l.nfaMatches = nil l.nfaMatches = nil
return fmt.Errorf("One or more of the supplied patterns match the empty string") return fmt.Errorf("one or more of the supplied patterns match the empty string")
} }
return nil return nil
@ -343,7 +349,7 @@ func (l *Lexer) CompileNFA() error {
// they are created. // they are created.
func (l *Lexer) CompileDFA() error { func (l *Lexer) CompileDFA() error {
if len(l.patterns) == 0 { if len(l.patterns) == 0 {
return fmt.Errorf("No patterns added") return fmt.Errorf("no patterns added")
} }
if l.dfa != nil { if l.dfa != nil {
return nil return nil
@ -363,7 +369,7 @@ func (l *Lexer) CompileDFA() error {
} else if mes { } else if mes {
l.dfa = nil l.dfa = nil
l.dfaMatches = nil l.dfaMatches = nil
return fmt.Errorf("One or more of the supplied patterns match the empty string") return fmt.Errorf("one or more of the supplied patterns match the empty string")
} }
return nil return nil
} }

30
lexer_test.go
View File

@ -95,21 +95,21 @@ func TestSimple(x *testing.T) {
`) `)
expected := []*Token{ expected := []*Token{
{NAME, "name", []byte("name"), 3, 2, 3, 2, 6}, {NAME, "name", []byte("name"), 3, 2, 3, 2, 6, 2, 3, 2, 6},
{EQUALS, nil, []byte("="), 8, 2, 8, 2, 8}, {EQUALS, nil, []byte("="), 8, 2, 8, 2, 8, 2, 8, 2, 8},
{NUMBER, 10, []byte("10"), 10, 2, 10, 2, 11}, {NUMBER, 10, []byte("10"), 10, 2, 10, 2, 11, 2, 10, 2, 11},
{PRINT, nil, []byte("print"), 15, 3, 3, 3, 7}, {PRINT, nil, []byte("print"), 15, 3, 3, 3, 7, 3, 3, 3, 7},
{NAME, "name", []byte("name"), 21, 3, 9, 3, 12}, {NAME, "name", []byte("name"), 21, 3, 9, 3, 12, 3, 9, 3, 12},
{PRINT, nil, []byte("print"), 28, 4, 3, 4, 7}, {PRINT, nil, []byte("print"), 28, 4, 3, 4, 7, 4, 3, 4, 7},
{NAME, "fred", []byte("fred"), 34, 4, 9, 4, 12}, {NAME, "fred", []byte("fred"), 34, 4, 9, 4, 12, 4, 9, 4, 12},
{NAME, "name", []byte("name"), 41, 5, 3, 5, 6}, {NAME, "name", []byte("name"), 41, 5, 3, 5, 6, 5, 3, 5, 6},
{EQUALS, nil, []byte("="), 46, 5, 8, 5, 8}, {EQUALS, nil, []byte("="), 46, 5, 8, 5, 8, 5, 8, 5, 8},
{NUMBER, 12, []byte("12"), 47, 5, 9, 5, 10}, {NUMBER, 12, []byte("12"), 47, 5, 9, 5, 10, 5, 9, 5, 10},
{NAME, "printname", []byte("printname"), 112, 9, 11, 9, 19}, {NAME, "printname", []byte("printname"), 112, 9, 11, 9, 19, 9, 11, 9, 19},
{EQUALS, nil, []byte("="), 122, 9, 21, 9, 21}, {EQUALS, nil, []byte("="), 122, 9, 21, 9, 21, 9, 21, 9, 21},
{NUMBER, 13, []byte("13"), 124, 9, 23, 9, 24}, {NUMBER, 13, []byte("13"), 124, 9, 23, 9, 24, 9, 23, 9, 24},
{PRINT, nil, []byte("print"), 129, 10, 3, 10, 7}, {PRINT, nil, []byte("print"), 129, 10, 3, 10, 7, 10, 3, 10, 7},
{NAME, "printname", []byte("printname"), 135, 10, 9, 10, 17}, {NAME, "printname", []byte("printname"), 135, 10, 9, 10, 17, 10, 9, 10, 17},
} }
scan := func(lexer *Lexer) { scan := func(lexer *Lexer) {