Compare commits
No commits in common. "main" and "dev" have entirely different histories.
@ -163,4 +163,4 @@ for tok, err, eof := s.Next(); !eof; tok, err, eof = s.Next() {
|
||||
}
|
||||
```
|
||||
|
||||
//TODO 错误信息的行列计算
|
||||
// TODO 优化位置标记计算
|
||||
@ -23,7 +23,10 @@ func TestParse(x *testing.T) {
|
||||
}
|
||||
|
||||
func tMatch(program inst.Slice, text string, t *test.T) {
|
||||
expected := []machines.Match{{PC: len(program) - 1, TC: 0, Bytes: []byte(text)}}
|
||||
expected := []machines.Match{{PC: len(program) - 1, TC: 0, StartLine: 1, StartColumn: 1, EndLine: 1, EndColumn: len(text), Bytes: []byte(text), TSLine: 1, TSColumn: 0, TELine: 1, TEColumn: 1}}
|
||||
if expected[0].EndColumn == 0 {
|
||||
expected[0].EndColumn = 1
|
||||
}
|
||||
i := 0
|
||||
scan := machines.LexerEngine(program, []byte(text))
|
||||
for tc, m, err, scan := scan(0); scan != nil; tc, m, err, scan = scan(tc) {
|
||||
|
||||
89
lexer.go
89
lexer.go
@ -3,7 +3,6 @@ package lexmachine
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"unicode/utf8"
|
||||
|
||||
dfapkg "gitea.xintech.co/zhouzhihong/lexmachine/dfa"
|
||||
@ -32,6 +31,8 @@ type Token struct {
|
||||
StartColumn int
|
||||
EndLine int
|
||||
EndColumn int
|
||||
|
||||
TSLine, TSColumn, TELine, TEColumn int
|
||||
}
|
||||
|
||||
// Equals checks the equality of two tokens ignoring the Value field.
|
||||
@ -55,7 +56,7 @@ func (t *Token) Equals(other *Token) bool {
|
||||
// String formats the token in a human readable form.
|
||||
func (t *Token) String() string {
|
||||
return fmt.Sprintf(
|
||||
"%d %q %d (%d, %d)-(%d, %d)",
|
||||
"%d %q %d (%d, %d)-(%d, %d) (%d, %d)-(%d, %d)",
|
||||
t.Type,
|
||||
t.Value,
|
||||
t.TC,
|
||||
@ -63,6 +64,10 @@ func (t *Token) String() string {
|
||||
t.StartColumn,
|
||||
t.EndLine,
|
||||
t.EndColumn,
|
||||
t.TSLine,
|
||||
t.TSColumn,
|
||||
t.TELine,
|
||||
t.TEColumn,
|
||||
)
|
||||
}
|
||||
|
||||
@ -119,6 +124,11 @@ type Scanner struct {
|
||||
Text []byte
|
||||
TC int
|
||||
pTC int
|
||||
sLine int
|
||||
sColumn int
|
||||
eLine int
|
||||
eColumn int
|
||||
|
||||
lpp map[int]lastPostion
|
||||
}
|
||||
|
||||
@ -151,7 +161,7 @@ type lastPostion struct {
|
||||
func (s *Scanner) Next() (tok interface{}, err error, eos bool) {
|
||||
var token interface{}
|
||||
for token == nil {
|
||||
ntc, match, err, scan := s.scan(s.TC)
|
||||
tc, match, err, scan := s.scan(s.TC)
|
||||
if scan == nil {
|
||||
return nil, nil, true
|
||||
} else if err != nil {
|
||||
@ -161,41 +171,50 @@ func (s *Scanner) Next() (tok interface{}, err error, eos bool) {
|
||||
}
|
||||
s.scan = scan
|
||||
s.pTC = s.TC
|
||||
s.TC = ntc
|
||||
s.TC = tc
|
||||
s.sLine = match.StartLine
|
||||
s.sColumn = match.StartColumn
|
||||
s.eLine = match.EndLine
|
||||
s.eColumn = match.EndColumn
|
||||
|
||||
p := s.pTC
|
||||
l, c := s.lpp[p].l, s.lpp[p].c
|
||||
stc := s.TC - len(match.Bytes)
|
||||
|
||||
for {
|
||||
if s.Text[p] == '\n' {
|
||||
l++
|
||||
c = 0
|
||||
} else {
|
||||
c++
|
||||
}
|
||||
|
||||
if p == stc {
|
||||
match.TSLine = l
|
||||
match.TSColumn = c
|
||||
}
|
||||
|
||||
match.TELine = l
|
||||
match.TEColumn = c
|
||||
|
||||
_, sz := utf8.DecodeRune(s.Text[p:])
|
||||
p += sz
|
||||
if p >= s.TC {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
s.lpp[s.TC] = lastPostion{
|
||||
l: l,
|
||||
c: c,
|
||||
}
|
||||
|
||||
pattern := s.lexer.patterns[s.matches[match.PC]]
|
||||
token, err = pattern.action(s, match)
|
||||
|
||||
lpp := s.lpp[s.pTC]
|
||||
line, col := lpp.l, lpp.c
|
||||
for i := s.pTC; i < s.TC; {
|
||||
ch, sz := utf8.DecodeRune(s.Text[i:])
|
||||
if ch == rune('\n') {
|
||||
line++
|
||||
col = 0
|
||||
} else {
|
||||
col++
|
||||
}
|
||||
|
||||
if (i == match.TC) && reflect.TypeOf(token) == reflect.TypeOf(&Token{}) {
|
||||
token.(*Token).StartLine = line
|
||||
token.(*Token).StartColumn = col
|
||||
}
|
||||
i += sz
|
||||
}
|
||||
|
||||
if reflect.TypeOf(token) == reflect.TypeOf(&Token{}) {
|
||||
token.(*Token).EndLine = line
|
||||
token.(*Token).EndColumn = col
|
||||
}
|
||||
|
||||
s.lpp[s.TC] = lastPostion{l: line, c: col}
|
||||
|
||||
if err != nil {
|
||||
return nil, err, false
|
||||
}
|
||||
}
|
||||
|
||||
return token, nil, false
|
||||
}
|
||||
|
||||
@ -206,6 +225,14 @@ func (s *Scanner) Token(typ int, value interface{}, m *machines.Match) *Token {
|
||||
Value: value,
|
||||
Lexeme: m.Bytes,
|
||||
TC: m.TC,
|
||||
StartLine: m.StartLine,
|
||||
StartColumn: m.StartColumn,
|
||||
EndLine: m.EndLine,
|
||||
EndColumn: m.EndColumn,
|
||||
TSLine: m.TSLine,
|
||||
TSColumn: m.TSColumn,
|
||||
TELine: m.TELine,
|
||||
TEColumn: m.TEColumn,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -77,8 +77,8 @@ func TestSimple(x *testing.T) {
|
||||
}
|
||||
}
|
||||
return nil,
|
||||
fmt.Errorf("unclosed comment starting at %d",
|
||||
match.TC)
|
||||
fmt.Errorf("unclosed comment starting at %d, (%d, %d)",
|
||||
match.TC, match.StartLine, match.StartColumn)
|
||||
},
|
||||
)
|
||||
|
||||
@ -95,22 +95,23 @@ func TestSimple(x *testing.T) {
|
||||
`)
|
||||
|
||||
expected := []*Token{
|
||||
{NAME, "name", []byte("name"), 3, 2, 3, 2, 6},
|
||||
{EQUALS, nil, []byte("="), 8, 2, 8, 2, 8},
|
||||
{NUMBER, 10, []byte("10"), 10, 2, 10, 2, 11},
|
||||
{PRINT, nil, []byte("print"), 15, 3, 3, 3, 7},
|
||||
{NAME, "name", []byte("name"), 21, 3, 9, 3, 12},
|
||||
{PRINT, nil, []byte("print"), 28, 4, 3, 4, 7},
|
||||
{NAME, "fred", []byte("fred"), 34, 4, 9, 4, 12},
|
||||
{NAME, "name", []byte("name"), 41, 5, 3, 5, 6},
|
||||
{EQUALS, nil, []byte("="), 46, 5, 8, 5, 8},
|
||||
{NUMBER, 12, []byte("12"), 47, 5, 9, 5, 10},
|
||||
{NAME, "printname", []byte("printname"), 112, 9, 11, 9, 19},
|
||||
{EQUALS, nil, []byte("="), 122, 9, 21, 9, 21},
|
||||
{NUMBER, 13, []byte("13"), 124, 9, 23, 9, 24},
|
||||
{PRINT, nil, []byte("print"), 129, 10, 3, 10, 7},
|
||||
{NAME, "printname", []byte("printname"), 135, 10, 9, 10, 17},
|
||||
{NAME, "name", []byte("name"), 3, 2, 3, 2, 6, 2, 3, 2, 6},
|
||||
{EQUALS, nil, []byte("="), 8, 2, 8, 2, 8, 2, 8, 2, 8},
|
||||
{NUMBER, 10, []byte("10"), 10, 2, 10, 2, 11, 2, 10, 2, 11},
|
||||
{PRINT, nil, []byte("print"), 15, 3, 3, 3, 7, 3, 3, 3, 7},
|
||||
{NAME, "name", []byte("name"), 21, 3, 9, 3, 12, 3, 9, 3, 12},
|
||||
{PRINT, nil, []byte("print"), 28, 4, 3, 4, 7, 4, 3, 4, 7},
|
||||
{NAME, "fred", []byte("fred"), 34, 4, 9, 4, 12, 4, 9, 4, 12},
|
||||
{NAME, "name", []byte("name"), 41, 5, 3, 5, 6, 5, 3, 5, 6},
|
||||
{EQUALS, nil, []byte("="), 46, 5, 8, 5, 8, 5, 8, 5, 8},
|
||||
{NUMBER, 12, []byte("12"), 47, 5, 9, 5, 10, 5, 9, 5, 10},
|
||||
{NAME, "printname", []byte("printname"), 112, 9, 11, 9, 19, 9, 11, 9, 19},
|
||||
{EQUALS, nil, []byte("="), 122, 9, 21, 9, 21, 9, 21, 9, 21},
|
||||
{NUMBER, 13, []byte("13"), 124, 9, 23, 9, 24, 9, 23, 9, 24},
|
||||
{PRINT, nil, []byte("print"), 129, 10, 3, 10, 7, 10, 3, 10, 7},
|
||||
{NAME, "printname", []byte("printname"), 135, 10, 9, 10, 17, 10, 9, 10, 17},
|
||||
}
|
||||
|
||||
scan := func(lexer *Lexer) {
|
||||
scanner, err := lexer.Scanner(text)
|
||||
if err != nil {
|
||||
@ -291,7 +292,6 @@ func TestRegression(t *testing.T) {
|
||||
fmt.Printf("Token: %v\n", tok)
|
||||
found++
|
||||
}
|
||||
|
||||
if found != test.tokens {
|
||||
t.Errorf("Expected exactly %v tokens got %v, ===\nErr: %v\nEOS: %v\nTC: %d\n", test.tokens, found, err, eos, scanner.TC)
|
||||
}
|
||||
|
||||
@ -34,6 +34,7 @@ func mapLineCols(text []byte) []lineCol {
|
||||
// DFA state machine. If the lexing process fails the Scanner will return
|
||||
// an UnconsumedInput error.
|
||||
func DFALexerEngine(startState, errorState int, trans DFATrans, accepting DFAAccepting, text []byte) Scanner {
|
||||
lineCols := mapLineCols(text)
|
||||
done := false
|
||||
matchID := -1
|
||||
matchTC := -1
|
||||
@ -61,15 +62,23 @@ func DFALexerEngine(startState, errorState int, trans DFATrans, accepting DFAAcc
|
||||
}
|
||||
state = trans[state][text[tc]]
|
||||
if state == errorState && matchID > -1 {
|
||||
startLC := lineCols[startTC]
|
||||
endLC := lineCols[matchTC-1]
|
||||
match := &Match{
|
||||
PC: matchID,
|
||||
TC: startTC,
|
||||
StartLine: startLC.line,
|
||||
StartColumn: startLC.col,
|
||||
EndLine: endLC.line,
|
||||
EndColumn: endLC.col,
|
||||
Bytes: text[startTC:matchTC],
|
||||
}
|
||||
if matchTC == startTC {
|
||||
err := &EmptyMatchError{
|
||||
MatchID: matchID,
|
||||
TC: startTC,
|
||||
TC: tc,
|
||||
Line: startLC.line,
|
||||
Column: startLC.col,
|
||||
}
|
||||
return startTC, nil, err, scan
|
||||
}
|
||||
@ -82,18 +91,29 @@ func DFALexerEngine(startState, errorState int, trans DFATrans, accepting DFAAcc
|
||||
matchTC = tc
|
||||
}
|
||||
if startTC <= len(text) && matchID > -1 && matchTC == startTC {
|
||||
|
||||
var startLC lineCol
|
||||
if startTC < len(text) {
|
||||
startLC = lineCols[startTC]
|
||||
}
|
||||
err := &EmptyMatchError{
|
||||
MatchID: matchID,
|
||||
TC: startTC,
|
||||
TC: tc,
|
||||
Line: startLC.line,
|
||||
Column: startLC.col,
|
||||
}
|
||||
matchID = -1
|
||||
return startTC, nil, err, scan
|
||||
}
|
||||
if startTC < len(text) && matchTC <= len(text) && matchID > -1 {
|
||||
startLC := lineCols[startTC]
|
||||
endLC := lineCols[matchTC-1]
|
||||
match := &Match{
|
||||
PC: matchID,
|
||||
TC: startTC,
|
||||
StartLine: startLC.line,
|
||||
StartColumn: startLC.col,
|
||||
EndLine: endLC.line,
|
||||
EndColumn: endLC.col,
|
||||
Bytes: text[startTC:matchTC],
|
||||
}
|
||||
matchID = -1
|
||||
@ -108,11 +128,21 @@ func DFALexerEngine(startState, errorState int, trans DFATrans, accepting DFAAcc
|
||||
if matchTC == -1 {
|
||||
matchTC = 0
|
||||
}
|
||||
startLC := lineCols[startTC]
|
||||
etc := tc
|
||||
|
||||
var endLC lineCol
|
||||
if etc >= len(lineCols) {
|
||||
endLC = lineCols[len(lineCols)-1]
|
||||
} else {
|
||||
endLC = lineCols[etc]
|
||||
}
|
||||
err := &UnconsumedInput{
|
||||
StartTC: startTC,
|
||||
FailTC: etc,
|
||||
StartLine: startLC.line,
|
||||
StartColumn: startLC.col,
|
||||
FailLine: endLC.line,
|
||||
FailColumn: endLC.col,
|
||||
Text: text,
|
||||
}
|
||||
return tc, nil, err, scan
|
||||
|
||||
@ -13,12 +13,14 @@ import (
|
||||
// string
|
||||
type EmptyMatchError struct {
|
||||
TC int
|
||||
Line int
|
||||
Column int
|
||||
MatchID int
|
||||
}
|
||||
|
||||
func (e *EmptyMatchError) Error() string {
|
||||
return fmt.Sprintf("Lexer error: matched the empty string at %d for match id %d.",
|
||||
e.TC, e.MatchID,
|
||||
return fmt.Sprintf("Lexer error: matched the empty string at %d:%d (tc=%d) for match id %d.",
|
||||
e.Line, e.Column, e.TC, e.MatchID,
|
||||
)
|
||||
}
|
||||
|
||||
@ -26,6 +28,10 @@ func (e *EmptyMatchError) Error() string {
|
||||
type UnconsumedInput struct {
|
||||
StartTC int
|
||||
FailTC int
|
||||
StartLine int
|
||||
StartColumn int
|
||||
FailLine int
|
||||
FailColumn int
|
||||
Text []byte
|
||||
}
|
||||
|
||||
@ -45,18 +51,60 @@ func (u *UnconsumedInput) Error() string {
|
||||
}
|
||||
stc := min(u.StartTC, len(u.Text)-1)
|
||||
etc := min(max(u.StartTC+1, u.FailTC), len(u.Text))
|
||||
return fmt.Sprintf("Lexer error: could not match text starting at %v failing at %v.\n\tunmatched text: %q",
|
||||
u.StartTC,
|
||||
u.FailTC,
|
||||
return fmt.Sprintf("Lexer error: could not match text starting at %v:%v failing at %v:%v.\n\tunmatched text: %q",
|
||||
u.StartLine, u.StartColumn,
|
||||
u.FailLine, u.FailColumn,
|
||||
string(u.Text[stc:etc]),
|
||||
)
|
||||
}
|
||||
|
||||
// A Match represents the positional and textual information from a match.
|
||||
type Match struct {
|
||||
PC int // pattern cursor
|
||||
TC int // start position of text cursor
|
||||
PC int
|
||||
TC int
|
||||
StartLine int
|
||||
StartColumn int
|
||||
EndLine int
|
||||
EndColumn int
|
||||
Bytes []byte // the actual bytes matched during scanning.
|
||||
|
||||
TSLine, TSColumn, TELine, TEColumn int
|
||||
}
|
||||
|
||||
func computeLineCol(text []byte, prevTC, tc, line, col int) (int, int) {
|
||||
if tc < 0 {
|
||||
return line, col
|
||||
}
|
||||
if tc < prevTC {
|
||||
for i := prevTC; i > tc && i > 0; i-- {
|
||||
if text[i] == '\n' {
|
||||
line--
|
||||
}
|
||||
}
|
||||
col = 0
|
||||
for i := tc; i >= 0; i-- {
|
||||
if text[i] == '\n' {
|
||||
break
|
||||
}
|
||||
col++
|
||||
}
|
||||
return line, col
|
||||
}
|
||||
for i := prevTC + 1; i <= tc && i < len(text); i++ {
|
||||
if text[i] == '\n' {
|
||||
col = 0
|
||||
line++
|
||||
} else {
|
||||
col++
|
||||
}
|
||||
}
|
||||
if prevTC == tc && tc == 0 && tc < len(text) {
|
||||
if text[tc] == '\n' {
|
||||
line++
|
||||
col--
|
||||
}
|
||||
}
|
||||
return line, col
|
||||
}
|
||||
|
||||
// Equals checks two matches for equality
|
||||
@ -69,13 +117,16 @@ func (m *Match) Equals(other *Match) bool {
|
||||
return false
|
||||
}
|
||||
return m.PC == other.PC &&
|
||||
m.TC == other.TC &&
|
||||
m.StartLine == other.StartLine &&
|
||||
m.StartColumn == other.StartColumn &&
|
||||
m.EndLine == other.EndLine &&
|
||||
m.EndColumn == other.EndColumn &&
|
||||
bytes.Equal(m.Bytes, other.Bytes)
|
||||
}
|
||||
|
||||
// String formats the match for humans
|
||||
func (m Match) String() string {
|
||||
return fmt.Sprintf("<Match %d %d %v'>", m.PC, m.TC, string(m.Bytes))
|
||||
return fmt.Sprintf("<Match %d %d (%d, %d)-(%d, %d) '%v'>", m.PC, m.TC, m.StartLine, m.StartColumn, m.EndLine, m.EndColumn, string(m.Bytes))
|
||||
}
|
||||
|
||||
// Scanner is a functional iterator returned by the LexerEngine. See
|
||||
@ -90,6 +141,10 @@ func LexerEngine(program inst.Slice, text []byte) Scanner {
|
||||
matchPC := -1
|
||||
matchTC := -1
|
||||
|
||||
prevTC := 0
|
||||
line := 1
|
||||
col := 1
|
||||
|
||||
var scan Scanner
|
||||
var cqueue, nqueue *queue.Queue = queue.New(len(program)), queue.New(len(program))
|
||||
scan = func(tc int) (int, *Match, error, Scanner) {
|
||||
@ -142,19 +197,27 @@ func LexerEngine(program inst.Slice, text []byte) Scanner {
|
||||
}
|
||||
cqueue, nqueue = nqueue, cqueue
|
||||
if cqueue.Empty() && matchPC > -1 {
|
||||
|
||||
line, col = computeLineCol(text, prevTC, startTC, line, col)
|
||||
eLine, eCol := computeLineCol(text, startTC, matchTC-1, line, col)
|
||||
match := &Match{
|
||||
PC: matchPC,
|
||||
TC: startTC,
|
||||
StartLine: line,
|
||||
StartColumn: col,
|
||||
EndLine: eLine,
|
||||
EndColumn: eCol,
|
||||
Bytes: text[startTC:matchTC],
|
||||
}
|
||||
if matchTC == startTC {
|
||||
err := &EmptyMatchError{
|
||||
MatchID: matchPC,
|
||||
TC: tc,
|
||||
Line: line,
|
||||
Column: col,
|
||||
}
|
||||
return startTC, nil, err, scan
|
||||
}
|
||||
prevTC = startTC
|
||||
matchPC = -1
|
||||
return matchTC, match, nil, scan
|
||||
}
|
||||
@ -168,9 +231,15 @@ func LexerEngine(program inst.Slice, text []byte) Scanner {
|
||||
if matchTC == -1 {
|
||||
matchTC = 0
|
||||
}
|
||||
sline, scol := computeLineCol(text, 0, startTC, 1, 1)
|
||||
fline, fcol := computeLineCol(text, 0, tc, 1, 1)
|
||||
err := &UnconsumedInput{
|
||||
StartTC: startTC,
|
||||
FailTC: tc,
|
||||
StartLine: sline,
|
||||
StartColumn: scol,
|
||||
FailLine: fline,
|
||||
FailColumn: fcol,
|
||||
Text: text,
|
||||
}
|
||||
return tc, nil, err, scan
|
||||
|
||||
@ -34,7 +34,7 @@ func TestLexerMatch(t *testing.T) {
|
||||
t.Log(program)
|
||||
mtext := []byte("ababcbcbb")
|
||||
expected := []Match{
|
||||
{16, 0, mtext},
|
||||
{16, 0, 1, 1, 1, len(mtext), mtext, 1, 0, 1, 1},
|
||||
}
|
||||
i := 0
|
||||
for tc, m, err, scan := LexerEngine(program, text)(0); scan != nil; tc, m, err, scan = scan(tc) {
|
||||
@ -114,9 +114,9 @@ func TestLexerThreeStrings(t *testing.T) {
|
||||
t.Log(len(text))
|
||||
t.Log(program)
|
||||
expected := []Match{
|
||||
{8, 0, []byte("struct")},
|
||||
{13, 6, []byte(" ")},
|
||||
{15, 8, []byte("*")},
|
||||
{8, 0, 1, 1, 1, 6, []byte("struct"), 1, 0, 1, 1},
|
||||
{13, 6, 1, 7, 1, 8, []byte(" "), 1, 0, 1, 1},
|
||||
{15, 8, 1, 9, 1, 9, []byte("*"), 1, 0, 1, 1},
|
||||
}
|
||||
|
||||
i := 0
|
||||
@ -165,9 +165,9 @@ func TestLexerRestart(t *testing.T) {
|
||||
t.Log(len(text))
|
||||
t.Log(program)
|
||||
expected := []Match{
|
||||
{8, 0, []byte("struct")},
|
||||
{19, 6, []byte("\n ")},
|
||||
{21, 9, []byte("*")},
|
||||
{8, 0, 1, 1, 1, 6, []byte("struct"), 1, 0, 1, 1},
|
||||
{19, 6, 2, 0, 2, 2, []byte("\n "), 1, 0, 1, 1},
|
||||
{21, 9, 2, 3, 2, 3, []byte("*"), 1, 0, 1, 1},
|
||||
}
|
||||
|
||||
check := func(m *Match, i int, err error) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user