adoc/token/lexer.go

package token

import (
	"bufio"
	"bytes"
	"io"
)

type Lexer struct {
	in  *bufio.Reader
	out chan Token
	err error
}

func NewLexer(in io.Reader) *Lexer {
	lex := &Lexer{
		in:  bufio.NewReader(in),
		out: make(chan Token),
	}

	go lex.run()

	return lex
}

func (lex *Lexer) Out() chan Token {
	return lex.out
}

func (lex *Lexer) Error() error {
	return lex.err
}

func (lex *Lexer) run() {
	for {
		tok := lex.scan()
		if tok.Type == TYPE_EOF {
			lex.out <- TOKEN_EOF
			close(lex.out)
			return
		}

		lex.out <- tok
	}
}

func (lex *Lexer) scan() Token {
	var buf bytes.Buffer

	ch := lex.read()
	if _, err := buf.WriteRune(ch); err != nil {
		lex.err = err
		return Token{Type: TYPE_EOF}
	}

	tt := FindTokenType(ch)

LEXSCANLOOP:
	for {
		ch = lex.read()
		switch {
		case ch == eof:
			break LEXSCANLOOP
		case FindTokenType(ch) != tt:
			lex.unread()
			break LEXSCANLOOP
		default:
			if _, err := buf.WriteRune(ch); err != nil {
				lex.err = err
				return Token{Type: TYPE_EOF}
			}
		}
	}

	return Token{
		Type:    tt,
		Literal: buf.String(),
	}
}

func (lex *Lexer) read() rune {
	ch, _, err := lex.in.ReadRune()
	if err != nil {
		return eof
	}

	return ch
}

func (lex *Lexer) unread() {
	lex.in.UnreadRune()
}

func FindTokenType(ch rune) TokenType {
	for tt, rr := range TokenTypeMap {
		for _, r := range rr {
			if ch == r {
				return tt
			}
		}
	}

	return TYPE_WORD
}