146 lines
2.3 KiB
Go
146 lines
2.3 KiB
Go
package token
|
|
|
|
const (
|
|
TYPE_EOF TokenType = iota
|
|
TYPE_EOS
|
|
TYPE_WHITESPACE
|
|
TYPE_NEWLINE
|
|
|
|
TYPE_EQUALSIGN
|
|
TYPE_ASTERISK
|
|
TYPE_UNDERSCORE
|
|
TYPE_BACKTICK
|
|
TYPE_DASH
|
|
TYPE_COLON
|
|
TYPE_BRACKETOPEN
|
|
TYPE_BRACKETCLOSE
|
|
|
|
TYPE_WORD
|
|
)
|
|
|
|
var (
|
|
eof = rune(0)
|
|
)
|
|
|
|
var TokenTypeMap = map[TokenType][]rune{
|
|
TYPE_EOF: {eof},
|
|
TYPE_WHITESPACE: {' ', '\t'},
|
|
TYPE_NEWLINE: {'\n'},
|
|
TYPE_EQUALSIGN: {'='},
|
|
TYPE_BACKTICK: {'`'},
|
|
TYPE_DASH: {'-'},
|
|
TYPE_COLON: {':'},
|
|
TYPE_ASTERISK: {'*'},
|
|
TYPE_UNDERSCORE: {'_'},
|
|
TYPE_BRACKETOPEN: {'['},
|
|
TYPE_BRACKETCLOSE: {']'},
|
|
}
|
|
|
|
type TokenType int
|
|
|
|
var (
|
|
TOKEN_EOF = Token{Type: TYPE_EOF}
|
|
TOKEN_EOS = Token{Type: TYPE_EOS}
|
|
TOKEN_NEWLINE = Token{Type: TYPE_NEWLINE, Literal: "\n"}
|
|
TOKEN_DOUBLE_NEWLINE = Token{Type: TYPE_NEWLINE, Literal: "\n\n"}
|
|
TOKEN_ASTERISK = Token{Type: TYPE_ASTERISK, Literal: "*"}
|
|
)
|
|
|
|
type Token struct {
|
|
Type TokenType
|
|
Literal string
|
|
}
|
|
|
|
func (t Token) Len() int {
|
|
return len(t.Literal)
|
|
}
|
|
|
|
func (t Token) Equal(wanted ...Token) bool {
|
|
for _, w := range wanted {
|
|
if t.Type == w.Type && t.Literal == w.Literal {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func Literals(ts []Token) string {
|
|
var s string
|
|
for _, t := range ts {
|
|
s += t.Literal
|
|
}
|
|
|
|
return s
|
|
}
|
|
|
|
func StartsWith(ts []Token, pat []Token) bool {
|
|
if len(ts) < len(pat) {
|
|
return false
|
|
}
|
|
for i, t := range ts {
|
|
if !t.Equal(pat[i]) {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func EndsWith(ts []Token, pat []Token) bool {
|
|
if len(ts) < len(pat) {
|
|
return false
|
|
}
|
|
for i := 1; i <= len(pat); i++ {
|
|
if !ts[len(ts)-i].Equal(pat[len(pat)-i]) {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func HasPattern(ts []Token, pat []TokenType) bool {
|
|
if len(ts) != len(pat) {
|
|
return false
|
|
}
|
|
for i, t := range ts {
|
|
if t.Type != pat[i] {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func Split(ts []Token, tt TokenType) [][]Token {
|
|
lines := [][]Token{}
|
|
line := []Token{}
|
|
for _, t := range ts {
|
|
if t.Type == tt {
|
|
lines = append(lines, line)
|
|
line = []Token{}
|
|
continue
|
|
}
|
|
line = append(line, t)
|
|
}
|
|
lines = append(lines, line)
|
|
|
|
return lines
|
|
}
|
|
|
|
func SplitOnFirst(ts []Token, tt TokenType) ([]Token, []Token) {
|
|
if len(ts) < 2 {
|
|
return ts, []Token{}
|
|
}
|
|
|
|
first := ts[0].Type
|
|
for i, tok := range ts[1:] {
|
|
if tok.Type == first {
|
|
return ts[1:i], ts[i:]
|
|
}
|
|
}
|
|
|
|
return ts, []Token{}
|
|
}
|