src

lexer.go (4006B)

---

 package pattern
 
 import (
 	"fmt"
 	"go/token"
 	"iter"
 	"unicode"
 	"unicode/utf8"
 )
 
 // lex returns the sequence of tokens in the input.
 func lex(f *token.File, input string) iter.Seq[item] {
 	return func(yield func(item) bool) {
 		lex := &lexer{
 			f:     f,
 			input: input,
 			yield: yield,
 		}
 		lex.run()
 	}
 }
 
 // lexer holds the state of a single [lex] iteration.
 type lexer struct {
 	f *token.File
 
 	input string
 	start int
 	pos   int
 	width int
 
 	yield func(item) bool
 }
 
 type itemType int
 
 const eof = -1
 
 const (
 	itemError itemType = iota
 	itemLeftParen
 	itemRightParen
 	itemLeftBracket
 	itemRightBracket
 	itemTypeName
 	itemVariable
 	itemAt
 	itemColon
 	itemBlank
 	itemString
 	itemEOF
 )
 
 func (typ itemType) String() string {
 	switch typ {
 	case itemError:
 		return "ERROR"
 	case itemLeftParen:
 		return "("
 	case itemRightParen:
 		return ")"
 	case itemLeftBracket:
 		return "["
 	case itemRightBracket:
 		return "]"
 	case itemTypeName:
 		return "TYPE"
 	case itemVariable:
 		return "VAR"
 	case itemAt:
 		return "@"
 	case itemColon:
 		return ":"
 	case itemBlank:
 		return "_"
 	case itemString:
 		return "STRING"
 	case itemEOF:
 		return "EOF"
 	default:
 		return fmt.Sprintf("itemType(%d)", typ)
 	}
 }
 
 type item struct {
 	typ itemType
 	val string
 	pos int
 }
 
 type stateFn func(*lexer) stateFn
 
 func (l *lexer) run() {
 	for state := lexStart; state != nil; {
 		state = state(l)
 	}
 }
 
 func (l *lexer) emitValue(t itemType, value string) bool {
 	ok := l.yield(item{t, value, l.start})
 	l.start = l.pos
 	return ok
 }
 
 func (l *lexer) emit(t itemType) bool {
 	ok := l.yield(item{t, l.input[l.start:l.pos], l.start})
 	l.start = l.pos
 	return ok
 }
 
 func lexStart(l *lexer) stateFn {
 	switch r := l.next(); {
 	case r == eof:
 		_ = l.emit(itemEOF)
 		return nil
 	case unicode.IsSpace(r):
 		l.ignore()
 	case r == '(':
 		if !l.emit(itemLeftParen) {
 			return nil
 		}
 	case r == ')':
 		if !l.emit(itemRightParen) {
 			return nil
 		}
 	case r == '[':
 		if !l.emit(itemLeftBracket) {
 			return nil
 		}
 	case r == ']':
 		if !l.emit(itemRightBracket) {
 			return nil
 		}
 	case r == '@':
 		if !l.emit(itemAt) {
 			return nil
 		}
 	case r == ':':
 		if !l.emit(itemColon) {
 			return nil
 		}
 	case r == '_':
 		if !l.emit(itemBlank) {
 			return nil
 		}
 	case r == '"':
 		l.backup()
 		return lexString
 	case unicode.IsUpper(r):
 		l.backup()
 		return lexType
 	case unicode.IsLower(r):
 		l.backup()
 		return lexVariable
 	default:
 		return l.errorf("unexpected character %c", r)
 	}
 	return lexStart
 }
 
 func (l *lexer) next() (r rune) {
 	if l.pos >= len(l.input) {
 		l.width = 0
 		return eof
 	}
 	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
 
 	if r == '\n' {
 		l.f.AddLine(l.pos)
 	}
 
 	l.pos += l.width
 
 	return r
 }
 
 func (l *lexer) ignore() {
 	l.start = l.pos
 }
 
 func (l *lexer) backup() {
 	l.pos -= l.width
 }
 
 func (l *lexer) errorf(format string, args ...any) stateFn {
 	// TODO(dh): emit position information in errors
 	_ = l.yield(item{
 		itemError,
 		fmt.Sprintf(format, args...),
 		l.start,
 	})
 	return nil
 }
 
 func isAlphaNumeric(r rune) bool {
 	return r >= '0' && r <= '9' ||
 		r >= 'a' && r <= 'z' ||
 		r >= 'A' && r <= 'Z'
 }
 
 func lexString(l *lexer) stateFn {
 	l.next() // skip quote
 	escape := false
 
 	var runes []rune
 	for {
 		switch r := l.next(); r {
 		case eof:
 			return l.errorf("unterminated string")
 		case '"':
 			if !escape {
 				if !l.emitValue(itemString, string(runes)) {
 					return nil
 				}
 				return lexStart
 			} else {
 				runes = append(runes, '"')
 				escape = false
 			}
 		case '\\':
 			if escape {
 				runes = append(runes, '\\')
 				escape = false
 			} else {
 				escape = true
 			}
 		default:
 			runes = append(runes, r)
 		}
 	}
 }
 
 func lexType(l *lexer) stateFn {
 	l.next()
 	for {
 		if !isAlphaNumeric(l.next()) {
 			l.backup()
 			if !l.emit(itemTypeName) {
 				return nil
 			}
 			return lexStart
 		}
 	}
 }
 
 func lexVariable(l *lexer) stateFn {
 	l.next()
 	for {
 		if !isAlphaNumeric(l.next()) {
 			l.backup()
 			if !l.emit(itemVariable) {
 				return nil
 			}
 			return lexStart
 		}
 	}
 }