ewintr
/
adoc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

decouple packages

This commit is contained in:
Erik Winter 2022-03-15 15:16:15 +01:00 committed by Erik Winter
parent 0ada801623
commit d2b00c93e1
37 changed files with 1118 additions and 947 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
adoc.go
View File

@ -2,6 +2,8 @@ package adoc
import (
"time"
"ewintr.nl/adoc/element"
)
type ADoc struct {
@ -10,12 +12,12 @@ type ADoc struct {
Author string
Path string
Date time.Time
Content []Element
Content []element.Element
}
func NewADoc() *ADoc {
return &ADoc{
Attributes: map[string]string{},
Content: []Element{},
Content: []element.Element{},
}
}

43
codeblock.go
View File

@ -1,43 +0,0 @@
package adoc
type CodeBlock []Element
func (cb CodeBlock) Text() string {
txt := ""
for _, e := range cb {
txt += e.Text()
}
return txt
}
func (p *Parser) tryCodeBlock() bool {
delimiter := Token{Type: TYPE_DASH, Literal: "----"}
toks, ok := p.readN(2)
if !ok {
return false
}
if !toks[0].Equal(delimiter) || toks[1].Type != TYPE_NEWLINE {
p.unread(2)
return false
}
for {
tok, ok := p.read()
if !ok {
p.unread(len(toks))
return false
}
if tok.Equal(delimiter) {
break
}
toks = append(toks, tok)
}
cb := CodeBlock{}
for _, tok := range toks[2:] {
cb = append(cb, p.makePlain(tok))
}
p.appendElement(cb)
return true
}

26
element.go
View File

@ -1,26 +0,0 @@
package adoc
type Element interface {
Text() string
}
type Empty struct{}
func (e Empty) Text() string { return "" }
type Word string
func (w Word) Text() string { return string(w) }
type WhiteSpace string
func (ws WhiteSpace) Text() string { return string(ws) }
type Image struct {
Src string
Alt string
}
func (i Image) Text() string {
return i.Alt
}

54
element/codeblock.go Normal file
View File

@ -0,0 +1,54 @@
package element
import (
"ewintr.nl/adoc/token"
)
type CodeBlock []Element
func (cb CodeBlock) Text() string {
txt := ""
for _, e := range cb {
txt += e.Text()
}
return txt
}
func (cb CodeBlock) Append(els []Element) Element {
return CodeBlock{append(cb, els...)}
}
func NewCodeBlockFromTokens(p ReadUnreader) (ParseResult, bool) {
delimiter := token.Token{Type: token.TYPE_DASH, Literal: "----"}
toks, ok := p.Read(2)
if !ok {
return ParseResult{}, false
}
if !toks[0].Equal(delimiter) || toks[1].Type != token.TYPE_NEWLINE {
p.Unread(2)
return ParseResult{}, false
}
for {
ntoks, ok := p.Read(1)
if !ok {
p.Unread(len(toks))
return ParseResult{}, false
}
tok := ntoks[0]
if tok.Equal(delimiter) {
break
}
toks = append(toks, tok)
}
cb := CodeBlock{}
for _, tok := range toks[2:] {
cb = append(cb, MakePlain(tok))
}
return ParseResult{
Element: cb,
Inner: []token.Token{},
}, true
}

38
codeblock_test.go → element/codeblock_test.go
View File

@ -1,10 +1,12 @@
package adoc_test
package element_test
import (
"strings"
"testing"
"ewintr.nl/adoc"
"ewintr.nl/adoc/element"
"ewintr.nl/adoc/parser"
"ewintr.nl/go-kit/test"
)
@ -20,7 +22,7 @@ func TestCodeBlock(t *testing.T) {
----`,
exp: &adoc.ADoc{
Attributes: map[string]string{},
Content: []adoc.Element{adoc.CodeBlock{}},
Content: []element.Element{element.CodeBlock{}},
},
},
{
@ -32,11 +34,11 @@ more
----`,
exp: &adoc.ADoc{
Attributes: map[string]string{},
Content: []adoc.Element{adoc.CodeBlock{
adoc.Word("code"),
adoc.WhiteSpace("\n\n"),
adoc.Word("more"),
adoc.WhiteSpace("\n"),
Content: []element.Element{element.CodeBlock{
element.Word("code"),
element.WhiteSpace("\n\n"),
element.Word("more"),
element.WhiteSpace("\n"),
}},
},
},
@ -49,22 +51,22 @@ more
`,
exp: &adoc.ADoc{
Attributes: map[string]string{},
Content: []adoc.Element{
adoc.Paragraph{
adoc.Word("----"),
adoc.WhiteSpace("\n"),
adoc.Word("code"),
},
adoc.Paragraph{
adoc.Word("more"),
adoc.WhiteSpace("\n"),
},
Content: []element.Element{
element.Paragraph{[]element.Element{
element.Word("----"),
element.WhiteSpace("\n"),
element.Word("code"),
}},
element.Paragraph{[]element.Element{
element.Word("more"),
element.WhiteSpace("\n"),
}},
},
},
},
} {
t.Run(tc.name, func(t *testing.T) {
par := adoc.NewParser(strings.NewReader(tc.input))
par := parser.New(strings.NewReader(tc.input))
test.Equals(t, tc.exp, par.Parse())
})
}

31
element/element.go Normal file
View File

@ -0,0 +1,31 @@
package element
import "ewintr.nl/adoc/token"
type Element interface {
Text() string
Append([]Element) Element
}
type ParseResult struct {
Element Element
Inner []token.Token
}
type ReadUnreader interface {
Read(n int) ([]token.Token, bool)
Unread(n int) bool
}
type Empty struct{}
func (e Empty) Text() string { return "" }
type Image struct {
Src string
Alt string
}
func (i Image) Text() string {
return i.Alt
}

94
element/header.go Normal file
View File

@ -0,0 +1,94 @@
package element
import (
"time"
"ewintr.nl/adoc/token"
)
type Header struct {
Title string
Author string
Date time.Time
Attributes map[string]string
}
func (h Header) Text() string { return h.Title }
func (h Header) Append(_ []Element) Element { return h }
func NewHeaderFromTokens(tr ReadUnreader) (ParseResult, bool) {
toks, ok := tr.Read(2)
if !ok {
return ParseResult{}, false
}
if !toks[0].Equal(token.Token{Type: token.TYPE_EQUALSIGN, Literal: "="}) {
tr.Unread(2)
return ParseResult{}, false
}
if toks[1].Type != token.TYPE_WHITESPACE {
tr.Unread(2)
return ParseResult{}, false
}
for {
ntoks, ok := tr.Read(1)
if !ok {
tr.Unread(len(toks))
return ParseResult{}, false
}
if ntoks[0].Equal(token.TOKEN_EOF, token.TOKEN_DOUBLE_NEWLINE) {
break
}
toks = append(toks, ntoks[0])
}
h := &Header{
Attributes: map[string]string{},
}
lines := token.Split(toks[2:], token.TYPE_NEWLINE)
h.Title = token.Literals(lines[0])
for _, line := range lines[1:] {
switch {
case tryHeaderDate(h, line):
continue
case tryHeaderField(h, line):
continue
default:
h.Author = token.Literals(line)
}
}
return ParseResult{
Element: *h,
Inner: []token.Token{},
}, true
}
func tryHeaderField(h *Header, line []token.Token) bool {
if len(line) < 4 {
return false
}
pair := token.Split(line, token.TYPE_WHITESPACE)
if len(pair) != 2 {
return false
}
key, value := pair[0], pair[1]
if !token.HasPattern(key, []token.TokenType{token.TYPE_COLON, token.TYPE_WORD, token.TYPE_COLON}) {
return false
}
h.Attributes[key[1].Literal] = token.Literals(value)
return true
}
func tryHeaderDate(h *Header, line []token.Token) bool {
date, err := time.Parse("2006-01-02", token.Literals(line))
if err != nil {
return false
}
h.Date = date
return true
}

20
header_test.go → element/header_test.go
View File

@ -1,4 +1,4 @@
package adoc_test
package element_test
import (
"strings"
@ -6,6 +6,8 @@ import (
"time"
"ewintr.nl/adoc"
"ewintr.nl/adoc/element"
"ewintr.nl/adoc/parser"
"ewintr.nl/go-kit/test"
)
@ -21,7 +23,7 @@ func TestHeader(t *testing.T) {
exp: &adoc.ADoc{
Title: "Title",
Attributes: map[string]string{},
Content: []adoc.Element{},
Content: []element.Element{},
},
},
{
@ -46,18 +48,18 @@ First paragraph`,
"key1": "value1",
"key2": "value2",
},
Content: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Word("First"),
adoc.WhiteSpace(" "),
adoc.Word("paragraph"),
}),
Content: []element.Element{
element.Paragraph{[]element.Element{
element.Word("First"),
element.WhiteSpace(" "),
element.Word("paragraph"),
}},
},
},
},
} {
t.Run(tc.name, func(t *testing.T) {
par := adoc.NewParser(strings.NewReader(tc.input))
par := parser.New(strings.NewReader(tc.input))
test.Equals(t, tc.exp, par.Parse())
})
}

72
element/link.go Normal file
View File

@ -0,0 +1,72 @@
package element
import (
"ewintr.nl/adoc/token"
)
type Link struct {
URL string
Title string
}
func (l Link) Text() string { return l.Title }
func (l Link) Append(_ []Element) Element { return l }
func NewLinkFromTokens(tr ReadUnreader) (ParseResult, bool) {
tok, ok := tr.Read(1)
if !ok {
return ParseResult{}, false
}
tr.Unread(1)
if tok[0].Type != token.TYPE_WORD {
return ParseResult{}, false
}
url, title := []token.Token{}, []token.Token{}
lb := false
count := 0
for {
ntoks, ok := tr.Read(1)
if !ok {
tr.Unread(count)
return ParseResult{}, false
}
count++
tok := ntoks[0]
if tok.Equal(token.TOKEN_EOF, token.TOKEN_EOS, token.TOKEN_NEWLINE, token.TOKEN_DOUBLE_NEWLINE) {
tr.Unread(count)
return ParseResult{}, false
}
if !lb && tok.Type == token.TYPE_WHITESPACE {
tr.Unread(count)
return ParseResult{}, false
}
if tok.Equal(token.Token{Type: token.TYPE_BRACKETOPEN, Literal: "["}) {
lb = true
continue
}
if lb && tok.Equal(token.Token{Type: token.TYPE_BRACKETCLOSE, Literal: "]"}) {
break
}
if lb {
title = append(title, tok)
continue
}
url = append(url, tok)
}
if len(url) == 0 || !lb {
tr.Unread(count)
return ParseResult{}, false
}
link := Link{
URL: token.Literals(url),
Title: token.Literals(title),
}
return ParseResult{
Element: link,
Inner: []token.Token{},
}, true
}

36
link_test.go → element/link_test.go
View File

@ -1,10 +1,12 @@
package adoc_test
package element_test
import (
"strings"
"testing"
"ewintr.nl/adoc"
"ewintr.nl/adoc/element"
"ewintr.nl/adoc/parser"
"ewintr.nl/go-kit/test"
)
@ -12,41 +14,41 @@ func TestLink(t *testing.T) {
for _, tc := range []struct {
name string
input string
exp []adoc.Element
exp []element.Element
}{
{
name: "simple",
input: "a link[title] somewhere",
exp: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Word("a"),
adoc.WhiteSpace(" "),
adoc.Link{
exp: []element.Element{
element.Paragraph{Elements: []element.Element{
element.Word("a"),
element.WhiteSpace(" "),
element.Link{
URL: "link",
Title: "title",
},
adoc.WhiteSpace(" "),
adoc.Word("somewhere"),
}),
element.WhiteSpace(" "),
element.Word("somewhere"),
}},
},
},
{
name: "with underscore",
input: "check https://example.com/some_url[some url]",
exp: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Word("check"),
adoc.WhiteSpace(" "),
adoc.Link{
exp: []element.Element{
element.Paragraph{Elements: []element.Element{
element.Word("check"),
element.WhiteSpace(" "),
element.Link{
URL: "https://example.com/some_url",
Title: "some url",
},
}),
}},
},
},
} {
t.Run(tc.name, func(t *testing.T) {
par := adoc.NewParser(strings.NewReader(tc.input))
par := parser.New(strings.NewReader(tc.input))
exp := &adoc.ADoc{
Attributes: map[string]string{},
Content: tc.exp,

56
element/list.go Normal file
View File

@ -0,0 +1,56 @@
package element
import (
"ewintr.nl/adoc/token"
)
type List []ListItem
func (l List) Text() string {
txt := ""
for _, li := range l {
txt += li.Text()
}
return txt
}
func (l List) Append(els []Element) Element {
for _, el := range els {
if val, ok := el.(ListItem); ok {
l = append(l, val)
}
}
return l
}
func NewListFromTokens(tr ReadUnreader) (ParseResult, bool) {
toks, ok := tr.Read(2)
if !ok {
return ParseResult{}, false
}
if !toks[0].Equal(token.TOKEN_ASTERISK) || toks[1].Type != token.TYPE_WHITESPACE {
tr.Unread(2)
return ParseResult{}, false
}
tr.Unread(2)
toks = []token.Token{}
for {
ntoks, ok := tr.Read(1)
if !ok {
tr.Unread(len(toks))
return ParseResult{}, false
}
tok := ntoks[0]
if tok.Equal(token.TOKEN_DOUBLE_NEWLINE, token.TOKEN_EOF) {
break
}
toks = append(toks, tok)
}
return ParseResult{
Element: List{},
Inner: toks,
}, true
}

79
element/list_test.go Normal file
View File

@ -0,0 +1,79 @@
package element_test
import (
"strings"
"testing"
"ewintr.nl/adoc"
"ewintr.nl/adoc/element"
"ewintr.nl/adoc/parser"
"ewintr.nl/go-kit/test"
)
func TestList(t *testing.T) {
for _, tc := range []struct {
name string
input string
exp []element.Element
}{
{
name: "one item",
input: `* item 1`,
exp: []element.Element{
element.List([]element.ListItem{
{element.Word("item"), element.WhiteSpace(" "), element.Word("1")},
},
)},
},
{
name: "multiple",
input: `* item 1
* item 2
* item 3`,
exp: []element.Element{
element.List([]element.ListItem{
{element.Word("item"), element.WhiteSpace(" "), element.Word("1")},
{element.Word("item"), element.WhiteSpace(" "), element.Word("2")},
{element.Word("item"), element.WhiteSpace(" "), element.Word("3")},
})},
},
{
name: "double with pararaph",
input: `* item 1
* item 2
* item 3
and some text`,
exp: []element.Element{
element.List(
[]element.ListItem{
{element.Word("item"), element.WhiteSpace(" "), element.Word("1")},
},
),
element.List(
[]element.ListItem{
{element.Word("item"), element.WhiteSpace(" "), element.Word("2")},
{element.Word("item"), element.WhiteSpace(" "), element.Word("3")},
},
),
element.Paragraph{Elements: []element.Element{
element.Word("and"),
element.WhiteSpace(" "),
element.Word("some"),
element.WhiteSpace(" "),
element.Word("text"),
}},
},
},
} {
t.Run(tc.name, func(t *testing.T) {
par := parser.New(strings.NewReader(tc.input))
exp := &adoc.ADoc{
Attributes: map[string]string{},
Content: tc.exp,
}
test.Equals(t, exp, par.Parse())
})
}
}

54
element/listitem.go Normal file
View File

@ -0,0 +1,54 @@
package element
import (
"ewintr.nl/adoc/token"
)
type ListItem []Element
func (li ListItem) Text() string {
txt := ""
for _, e := range li {
txt += e.Text()
}
return txt
}
func (li ListItem) Append(els []Element) Element {
for _, el := range els {
li = append(li, el)
}
return li
}
func NewListItemFromTokens(tr ReadUnreader) (ParseResult, bool) {
toks, ok := tr.Read(2)
if !ok {
return ParseResult{}, false
}
if !toks[0].Equal(token.TOKEN_ASTERISK) || toks[1].Type != token.TYPE_WHITESPACE {
tr.Unread(2)
return ParseResult{}, false
}
toks = []token.Token{}
for {
ntoks, ok := tr.Read(1)
if !ok {
tr.Unread(len(toks))
return ParseResult{}, false
}
tok := ntoks[0]
if tok.Equal(token.TOKEN_NEWLINE, token.TOKEN_EOS) {
break
}
toks = append(toks, tok)
}
return ParseResult{
Element: ListItem{},
Inner: toks,
}, true
}

48
element/paragraph.go Normal file
View File

@ -0,0 +1,48 @@
package element
import (
"ewintr.nl/adoc/token"
)
type Paragraph struct {
Elements []Element
}
func (p Paragraph) Text() string {
txt := ""
for _, el := range p.Elements {
txt += el.Text()
}
return txt
}
func (p Paragraph) Append(els []Element) Element {
return Paragraph{
Elements: append(p.Elements, els...),
}
}
func NewParagraphFromTokens(tr ReadUnreader) (ParseResult, bool) {
toks := []token.Token{}
for {
tok, ok := tr.Read(1)
if !ok {
tr.Unread(len(toks))
return ParseResult{}, false
}
if tok[0].Equal(token.TOKEN_EOF, token.TOKEN_DOUBLE_NEWLINE) {
break
}
toks = append(toks, tok[0])
}
if len(toks) == 0 {
return ParseResult{}, false
}
return ParseResult{
Element: Paragraph{Elements: []Element{}},
Inner: toks,
}, true
}

78
element/paragraph_test.go Normal file
View File

@ -0,0 +1,78 @@
package element_test
import (
"strings"
"testing"
"ewintr.nl/adoc"
"ewintr.nl/adoc/element"
"ewintr.nl/adoc/parser"
"ewintr.nl/go-kit/test"
)
func TestParagraph(t *testing.T) {
for _, tc := range []struct {
name string
input string
exp *adoc.ADoc
}{
{
name: "single paragraph",
input: "some text",
exp: &adoc.ADoc{
Attributes: map[string]string{},
Content: []element.Element{
element.Paragraph{Elements: []element.Element{
element.Word("some"),
element.WhiteSpace(" "),
element.Word("text"),
}},
}},
},
{
name: "title with paragraphs",
input: `= Title
paragraph one
paragraph two`,
exp: &adoc.ADoc{
Title: "Title",
Attributes: map[string]string{},
Content: []element.Element{
element.Paragraph{Elements: []element.Element{
element.Word("paragraph"),
element.WhiteSpace(" "),
element.Word("one"),
}},
element.Paragraph{Elements: []element.Element{
element.Word("paragraph"),
element.WhiteSpace(" "),
element.Word("two"),
}},
},
},
},
{
name: "three with trailing newline",
input: `one
two
three
`,
exp: &adoc.ADoc{
Attributes: map[string]string{},
Content: []element.Element{
element.Paragraph{Elements: []element.Element{element.Word("one")}},
element.Paragraph{Elements: []element.Element{element.Word("two")}},
element.Paragraph{Elements: []element.Element{element.Word("three"), element.WhiteSpace("\n")}},
}},
},
} {
t.Run(tc.name, func(t *testing.T) {
par := parser.New(strings.NewReader(tc.input))
test.Equals(t, tc.exp, par.Parse())
})
}
}

25
element/plain.go Normal file
View File

@ -0,0 +1,25 @@
package element
import "ewintr.nl/adoc/token"
type Word string
func (w Word) Text() string { return string(w) }
func (w Word) Append(_ []Element) Element { return w }
type WhiteSpace string
func (ws WhiteSpace) Text() string { return string(ws) }
func (ws WhiteSpace) Append(_ []Element) Element { return ws }
func MakePlain(tok token.Token) Element {
switch tok.Type {
case token.TYPE_WHITESPACE:
return WhiteSpace(tok.Literal)
case token.TYPE_NEWLINE:
return WhiteSpace(tok.Literal)
default:
return Word(tok.Literal)
}
}

106
element/styles.go Normal file
View File

@ -0,0 +1,106 @@
package element
import "ewintr.nl/adoc/token"
type Strong []Element
func (st Strong) Text() string {
var txt string
for _, e := range st {
txt += e.Text()
}
return txt
}
func (st Strong) Append(els []Element) Element {
return append(st, els...)
}
type Emphasis []Element
func (em Emphasis) Text() string {
var txt string
for _, e := range em {
txt += e.Text()
}
return txt
}
func (em Emphasis) Append(els []Element) Element {
return append(em, els...)
}
type Code []Element
func (c Code) Text() string {
var txt string
for _, e := range c {
txt += e.Text()
}
return txt
}
func (c Code) Append(els []Element) Element {
return append(c, els...)
}
func NewStyleFromTokens(tr ReadUnreader) (ParseResult, bool) {
toks, ok := tr.Read(2)
if !ok {
return ParseResult{}, false
}
markers := []token.Token{
{Type: token.TYPE_ASTERISK, Literal: "*"},
{Type: token.TYPE_UNDERSCORE, Literal: "_"},
{Type: token.TYPE_BACKTICK, Literal: "`"},
}
if !toks[0].Equal(markers...) {
tr.Unread(2)
return ParseResult{}, false
}
if toks[1].Type == token.TYPE_WHITESPACE {
tr.Unread(2)
return ParseResult{}, false
}
marker := toks[0]
toks = []token.Token{toks[1]}
for {
ntoks, ok := tr.Read(1)
if !ok {
tr.Unread(len(toks) + 1)
return ParseResult{}, false
}
tok := ntoks[0]
if tok.Equal(token.TOKEN_EOF, token.TOKEN_DOUBLE_NEWLINE) {
tr.Unread(len(toks) + 1)
return ParseResult{}, false
}
if tok.Equal(marker) {
break
}
toks = append(toks, tok)
}
if toks[len(toks)-1].Type == token.TYPE_WHITESPACE {
tr.Unread(len(toks) + 2)
return ParseResult{}, false
}
var s Element
switch marker.Type {
case token.TYPE_ASTERISK:
s = Strong{}
case token.TYPE_UNDERSCORE:
s = Emphasis{}
case token.TYPE_BACKTICK:
s = Code{}
}
return ParseResult{
Element: s,
Inner: toks,
}, true
}

103
element/styles_test.go Normal file
View File

@ -0,0 +1,103 @@
package element_test
import (
"strings"
"testing"
"ewintr.nl/adoc"
"ewintr.nl/adoc/element"
"ewintr.nl/adoc/parser"
"ewintr.nl/go-kit/test"
)
func TestStyles(t *testing.T) {
for _, tc := range []struct {
name string
input string
exp []element.Element
}{
{
name: "strong",
input: "*strong*",
exp: []element.Element{
element.Paragraph{Elements: []element.Element{
element.Strong{element.Word("strong")},
},
},
},
},
{
name: "emphasis",
input: "_emphasis_",
exp: []element.Element{
element.Paragraph{Elements: []element.Element{
element.Emphasis{element.Word("emphasis")},
},
},
},
},
{
name: "code",
input: "`code`",
exp: []element.Element{
element.Paragraph{Elements: []element.Element{
element.Code{element.Word("code")},
},
},
},
},
{
name: "mixed",
input: "some `code code` in plain",
exp: []element.Element{
element.Paragraph{Elements: []element.Element{
element.Word("some"),
element.WhiteSpace(" "),
element.Code{
element.Word("code"),
element.WhiteSpace(" "),
element.Word("code"),
},
element.WhiteSpace(" "),
element.Word("in"),
element.WhiteSpace(" "),
element.Word("plain"),
},
},
},
},
{
name: "incomplete",
input: "a *word",
exp: []element.Element{
element.Paragraph{Elements: []element.Element{
element.Word("a"),
element.WhiteSpace(" "),
element.Word("*"),
element.Word("word"),
}},
},
},
{
name: "trailing space",
input: "*word *",
exp: []element.Element{
element.Paragraph{Elements: []element.Element{
element.Word("*"),
element.Word("word"),
element.WhiteSpace(" "),
element.Word("*"),
}},
},
},
} {
t.Run(tc.name, func(t *testing.T) {
par := parser.New(strings.NewReader(tc.input))
exp := &adoc.ADoc{
Attributes: map[string]string{},
Content: tc.exp,
}
test.Equals(t, exp, par.Parse())
})
}
}

62
element/subtitle.go Normal file
View File

@ -0,0 +1,62 @@
package element
import "ewintr.nl/adoc/token"
type SubTitle string
func (st SubTitle) Text() string { return string(st) }
func (st SubTitle) Append(_ []Element) Element { return st }
type SubSubTitle string
func (st SubSubTitle) Text() string { return string(st) }
func (st SubSubTitle) Append(_ []Element) Element { return st }
func NewSubTitleFromTokens(tr ReadUnreader) (ParseResult, bool) {
toks, ok := tr.Read(2)
if !ok {
return ParseResult{}, false
}
if toks[0].Type != token.TYPE_EQUALSIGN || toks[0].Len() < 2 || toks[1].Type != token.TYPE_WHITESPACE {
tr.Unread(2)
return ParseResult{}, false
}
for {
ntoks, ok := tr.Read(1)
if !ok {
tr.Unread(len(toks))
return ParseResult{}, false
}
tok := ntoks[0]
if (tok.Type == token.TYPE_NEWLINE && tok.Len() > 1) || tok.Equal(token.TOKEN_EOF) {
break
}
toks = append(toks, tok)
}
var title string
for _, tok := range toks[2:] {
if tok.Type == token.TYPE_NEWLINE {
continue
}
title += MakePlain(tok).Text()
}
var el Element
switch toks[0].Len() {
case 2:
el = SubTitle(title)
case 3:
el = SubSubTitle(title)
default:
// ignore lower levels for now
tr.Unread(len(toks))
return ParseResult{}, false
}
return ParseResult{
Element: el,
Inner: []token.Token{},
}, true
}

16
subtitle_test.go → element/subtitle_test.go
View File

@ -1,10 +1,12 @@
package adoc_test
package element_test
import (
"strings"
"testing"
"ewintr.nl/adoc"
"ewintr.nl/adoc/element"
"ewintr.nl/adoc/parser"
"ewintr.nl/go-kit/test"
)
@ -12,27 +14,27 @@ func TestSubTitle(t *testing.T) {
for _, tc := range []struct {
name string
input string
exp []adoc.Element
exp []element.Element
}{
{
name: "empty",
input: "== ",
exp: []adoc.Element{adoc.SubTitle("")},
exp: []element.Element{element.SubTitle("")},
},
{
name: "subtitle",
input: "== title with words",
exp: []adoc.Element{adoc.SubTitle("title with words")},
exp: []element.Element{element.SubTitle("title with words")},
},
{
name: "subsubtitle",
input: "=== title",
exp: []adoc.Element{adoc.SubSubTitle("title")},
exp: []element.Element{element.SubSubTitle("title")},
},
{
name: "trailing newline",
input: "== title\n",
exp: []adoc.Element{adoc.SubTitle("title")},
exp: []element.Element{element.SubTitle("title")},
},
} {
t.Run(tc.name, func(t *testing.T) {
@ -40,7 +42,7 @@ func TestSubTitle(t *testing.T) {
Attributes: map[string]string{},
Content: tc.exp,
}
par := adoc.NewParser(strings.NewReader(tc.input))
par := parser.New(strings.NewReader(tc.input))
test.Equals(t, exp, par.Parse())
})
}

76
header.go
View File

@ -1,76 +0,0 @@
package adoc
import (
"time"
)
func (p *Parser) tryHeader() bool {
toks, ok := p.readN(2)
if !ok {
return false
}
if !toks[0].Equal(Token{Type: TYPE_EQUALSIGN, Literal: "="}) {
p.unread(2)
return false
}
if toks[1].Type != TYPE_WHITESPACE {
p.unread(2)
return false
}
for {
tok, ok := p.read()
if !ok {
p.unread(len(toks))
return false
}
if tok.Equal(TOKEN_EOF, TOKEN_DOUBLE_NEWLINE) {
break
}
toks = append(toks, tok)
}
lines := Split(toks[2:], TYPE_NEWLINE)
p.doc.Title = Literals(lines[0])
for _, line := range lines[1:] {
switch {
case p.tryHeaderDate(line):
continue
case p.tryHeaderField(line):
continue
default:
p.doc.Author = Literals(line)
}
}
return true
}
func (p *Parser) tryHeaderField(line []Token) bool {
if len(line) < 4 {
return false
}
pair := Split(line, TYPE_WHITESPACE)
if len(pair) != 2 {
return false
}
key, value := pair[0], pair[1]
if !HasPattern(key, []TokenType{TYPE_COLON, TYPE_WORD, TYPE_COLON}) {
return false
}
p.doc.Attributes[key[1].Literal] = Literals(value)
return true
}
func (p *Parser) tryHeaderDate(line []Token) bool {
date, err := time.Parse("2006-01-02", Literals(line))
if err != nil {
return false
}
p.doc.Date = date
return true
}

65
link.go
View File

@ -1,65 +0,0 @@
package adoc
import "fmt"
type Link struct {
URL string
Title string
}
func (l Link) Text() string { return l.Title }
func (p *Parser) tryLink() bool {
tok, ok := p.peek()
if !ok {
return false
}
if tok.Type != TYPE_WORD {
return false
}
url, title := []Token{}, []Token{}
lb := false
count := 0
for {
tok, ok := p.read()
if !ok {
p.unread(count)
return false
}
count++
if tok.Equal(TOKEN_EOF, TOKEN_NEWLINE, TOKEN_DOUBLE_NEWLINE) {
p.unread(count)
return false
}
if !lb && tok.Type == TYPE_WHITESPACE {
p.unread(count)
return false
}
if tok.Equal(Token{Type: TYPE_BRACKETOPEN, Literal: "["}) {
lb = true
continue
}
if tok.Equal(Token{Type: TYPE_BRACKETCLOSE, Literal: "]"}) {
break
}
if lb {
title = append(title, tok)
continue
}
url = append(url, tok)
}
fmt.Printf("url: %+v, title: %+v\n", url, title)
if len(url) == 0 || !lb {
p.unread(count)
return false
}
link := Link{
URL: Literals(url),
Title: Literals(title),
}
p.appendElement(link)
return true
}

77
list.go
View File

@ -1,77 +0,0 @@
package adoc
type ListItem []Element
func (li ListItem) Text() string {
txt := ""
for _, e := range li {
txt += e.Text()
}
return txt
}
type List []ListItem
func (l List) Text() string {
txt := ""
for _, li := range l {
txt += li.Text()
}
return txt
}
func (p *Parser) tryList() bool {
toks, ok := p.readN(2)
if !ok {
return false
}
if !toks[0].Equal(TOKEN_ASTERISK) || toks[1].Type != TYPE_WHITESPACE {
p.unread(2)
return false
}
p.unread(2)
toks = []Token{}
toksCount := 0
var items []ListItem
for {
tok, ok := p.read()
if !ok {
p.unread(toksCount)
return false
}
if tok.Equal(TOKEN_NEWLINE, TOKEN_DOUBLE_NEWLINE, TOKEN_EOF) {
item, ok := tryListItem(toks)
if !ok {
p.unread(len(toks))
return false
}
items = append(items, item)
toks = []Token{}
if tok.Equal(TOKEN_DOUBLE_NEWLINE, TOKEN_EOF) {
break
}
continue
}
toks = append(toks, tok)
toksCount++
}
p.appendElement(List(items))
return true
}
func tryListItem(toks []Token) (ListItem, bool) {
if !toks[0].Equal(TOKEN_ASTERISK) || toks[1].Type != TYPE_WHITESPACE {
return ListItem{}, false
}
stream := NewTokenStream(toks[2:]).Out()
par := NewParserFromChannel(stream)
par.Parse()
b := ListItem(par.Elements())
return b, true
}

77
list_test.go
View File

@ -1,77 +0,0 @@
package adoc_test
import (
"strings"
"testing"
"ewintr.nl/adoc"
"ewintr.nl/go-kit/test"
)
func TestList(t *testing.T) {
for _, tc := range []struct {
name string
input string
exp []adoc.Element
}{
{
name: "one item",
input: `* item 1`,
exp: []adoc.Element{
adoc.List([]adoc.ListItem{
{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("1")},
},
)},
},
{
name: "multiple",
input: `* item 1
* item 2
* item 3`,
exp: []adoc.Element{
adoc.List([]adoc.ListItem{
{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("1")},
{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("2")},
{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("3")},
})},
},
{
name: "double with pararaph",
input: `* item 1
* item 2
* item 3
and some text`,
exp: []adoc.Element{
adoc.List(
[]adoc.ListItem{
{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("1")},
},
),
adoc.List(
[]adoc.ListItem{
{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("2")},
{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("3")},
},
),
adoc.Paragraph([]adoc.Element{
adoc.Word("and"),
adoc.WhiteSpace(" "),
adoc.Word("some"),
adoc.WhiteSpace(" "),
adoc.Word("text"),
}),
},
},
} {
t.Run(tc.name, func(t *testing.T) {
par := adoc.NewParser(strings.NewReader(tc.input))
exp := &adoc.ADoc{
Attributes: map[string]string{},
Content: tc.exp,
}
test.Equals(t, exp, par.Parse())
})
}
}

39
paragraph.go
View File

@ -1,39 +0,0 @@
package adoc
type Paragraph []Element
func (p Paragraph) Text() string {
txt := ""
for _, e := range p {
txt += e.Text()
}
return txt
}
func (p *Parser) tryParagraph() bool {
toks := []Token{}
for {
tok, ok := p.read()
if !ok {
p.unread(len(toks))
return false
}
if tok.Equal(TOKEN_EOF, TOKEN_DOUBLE_NEWLINE) {
break
}
toks = append(toks, tok)
}
if len(toks) == 0 {
return false
}
par := NewParserFromChannel(NewTokenStream(toks).Out())
par.Parse()
b := Paragraph(par.Elements())
p.appendElement(b)
return true
}

76
paragraph_test.go
View File

@ -1,76 +0,0 @@
package adoc_test
import (
"strings"
"testing"
"ewintr.nl/adoc"
"ewintr.nl/go-kit/test"
)
func TestParagraph(t *testing.T) {
for _, tc := range []struct {
name string
input string
exp *adoc.ADoc
}{
{
name: "single paragraph",
input: "some text",
exp: &adoc.ADoc{
Attributes: map[string]string{},
Content: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Word("some"),
adoc.WhiteSpace(" "),
adoc.Word("text"),
}),
}},
},
{
name: "title with paragraphs",
input: `= Title
paragraph one
paragraph two`,
exp: &adoc.ADoc{
Title: "Title",
Attributes: map[string]string{},
Content: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Word("paragraph"),
adoc.WhiteSpace(" "),
adoc.Word("one"),
}),
adoc.Paragraph([]adoc.Element{
adoc.Word("paragraph"),
adoc.WhiteSpace(" "),
adoc.Word("two"),
}),
},
},
},
{
name: "three with trailing newline",
input: `one
two
three
`,
exp: &adoc.ADoc{
Attributes: map[string]string{},
Content: []adoc.Element{
adoc.Paragraph([]adoc.Element{adoc.Word("one")}),
adoc.Paragraph([]adoc.Element{adoc.Word("two")}),
adoc.Paragraph([]adoc.Element{adoc.Word("three"), adoc.WhiteSpace("\n")}),
}},
},
} {
t.Run(tc.name, func(t *testing.T) {
par := adoc.NewParser(strings.NewReader(tc.input))
test.Equals(t, tc.exp, par.Parse())
})
}
}

145
parser.go
View File

@ -1,145 +0,0 @@
package adoc
import "io"
type Parser struct {
in chan Token
doc *ADoc
ts []Token
r int
els []Element
}
func NewParser(reader io.Reader) *Parser {
lex := NewLexer(reader)
return NewParserFromChannel(lex.Out())
}
func NewParserFromChannel(toks chan Token) *Parser {
return &Parser{
in: toks,
doc: NewADoc(),
ts: []Token{},
els: []Element{},
}
}
func (p *Parser) Parse() *ADoc {
p.tryHeader()
for {
if done := p.scan(); done {
break
}
p.discard()
}
p.doc.Content = p.els
return p.doc
}
func (p *Parser) Elements() []Element {
return p.els
}
func (p *Parser) scan() bool {
if _, ok := p.peek(); !ok {
return true
}
type tryFunc func() bool
for _, tf := range []tryFunc{
p.tryCodeBlock, p.trySubTitle, p.tryList, p.tryParagraph,
p.tryStyles, p.tryLink,
} {
if tf() {
return false
}
}
p.doPlain()
return false
}
func (p *Parser) doPlain() {
tok, ok := p.read()
if !ok {
return
}
p.appendElement(p.makePlain(tok))
}
func (p *Parser) makePlain(tok Token) Element {
switch tok.Type {
case TYPE_WHITESPACE:
return WhiteSpace(tok.Literal)
case TYPE_NEWLINE:
return WhiteSpace(tok.Literal)
default:
return Word(tok.Literal)
}
}
func (p *Parser) appendElement(b Element) {
p.els = append(p.els, b)
}
func (p *Parser) consume() bool {
tok, ok := <-p.in
if !ok {
return false
}
p.ts = append(p.ts, tok)
return true
}
func (p *Parser) peek() (Token, bool) {
if p.r == len(p.ts) {
if ok := p.consume(); !ok {
return Token{}, false
}
}
return p.ts[p.r], true
}
func (p *Parser) readN(count int) ([]Token, bool) {
toks := []Token{}
for i := 0; i < count; i++ {
tok, ok := p.read()
if !ok {
p.unread(len(toks))
return []Token{}, false
}
toks = append(toks, tok)
}
return toks, true
}
func (p *Parser) read() (Token, bool) {
if p.r == len(p.ts) {
if ok := p.consume(); !ok {
return Token{}, false
}
}
tok := p.ts[p.r]
p.r++
return tok, true
}
func (p *Parser) unread(count int) {
for i := count; i > 0; i-- {
p.r--
}
}
func (p *Parser) discard() {
p.ts = p.ts[p.r:]
p.r = 0
}

101
parser/parser.go Normal file
View File

@ -0,0 +1,101 @@
package parser
import (
"io"
"ewintr.nl/adoc"
"ewintr.nl/adoc/element"
"ewintr.nl/adoc/token"
)
type Parser struct {
doc *adoc.ADoc
tr *token.TokenReader
els []element.Element
}
func New(reader io.Reader) *Parser {
lex := token.NewLexer(reader)
return NewParserFromChannel(lex.Out())
}
func NewParserFromChannel(toks chan token.Token) *Parser {
return &Parser{
doc: adoc.NewADoc(),
tr: token.NewTokenReader(toks),
els: []element.Element{},
}
}
func (p *Parser) Parse() *adoc.ADoc {
result, ok := element.NewHeaderFromTokens(p.tr)
if ok {
if h, ok := result.Element.(element.Header); ok {
p.doc.Title = h.Title
p.doc.Author = h.Author
p.doc.Date = h.Date
p.doc.Attributes = h.Attributes
}
}
for {
if done := p.scan(); done {
break
}
p.tr.Discard()
}
p.doc.Content = p.els
return p.doc
}
func (p *Parser) Elements() []element.Element {
return p.els
}
func (p *Parser) scan() bool {
if _, ok := p.tr.Read(1); !ok {
return true
}
p.tr.Unread(1)
type tryFunc func(element.ReadUnreader) (element.ParseResult, bool)
for _, tf := range []tryFunc{
element.NewCodeBlockFromTokens,
element.NewSubTitleFromTokens,
element.NewListFromTokens,
element.NewListItemFromTokens,
element.NewParagraphFromTokens,
element.NewStyleFromTokens,
element.NewLinkFromTokens,
} {
result, ok := tf(p.tr)
if !ok {
continue
}
el := result.Element
if len(result.Inner) != 0 {
par := NewParserFromChannel(token.NewTokenStream(result.Inner).Out())
par.Parse()
el = el.Append(par.Elements())
}
p.appendElement(el)
return false
}
p.doPlain()
return false
}
func (p *Parser) doPlain() {
tok, ok := p.tr.Read(1)
if !ok || tok[0].Type == token.TYPE_EOF || tok[0].Type == token.TYPE_EOS {
return
}
p.appendElement(element.MakePlain(tok[0]))
}
func (p *Parser) appendElement(b element.Element) {
p.els = append(p.els, b)
}

5
parser_test.go → parser/parser_test.go
View File

@ -1,10 +1,11 @@
package adoc_test
package parser_test
import (
"strings"
"testing"
"ewintr.nl/adoc"
"ewintr.nl/adoc/parser"
"ewintr.nl/go-kit/test"
)
@ -20,7 +21,7 @@ func TestParser(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
par := adoc.NewParser(strings.NewReader(tc.input))
par := parser.New(strings.NewReader(tc.input))
test.Equals(t, tc.exp, par.Parse())
})
}

92
styles.go
View File

@ -1,92 +0,0 @@
package adoc
type Strong []Element
func (st Strong) Text() string {
var txt string
for _, e := range st {
txt += e.Text()
}
return txt
}
type Emphasis []Element
func (em Emphasis) Text() string {
var txt string
for _, e := range em {
txt += e.Text()
}
return txt
}
type Code []Element
func (c Code) Text() string {
var txt string
for _, e := range c {
txt += e.Text()
}
return txt
}
func (p *Parser) tryStyles() bool {
tok, ok := p.readN(2)
if !ok {
return false
}
markers := []Token{
{Type: TYPE_ASTERISK, Literal: "*"},
{Type: TYPE_UNDERSCORE, Literal: "_"},
{Type: TYPE_BACKTICK, Literal: "`"},
}
if !tok[0].Equal(markers...) {
p.unread(2)
return false
}
if tok[1].Type == TYPE_WHITESPACE {
p.unread(2)
return false
}
marker := tok[0]
toks := []Token{tok[1]}
for {
tok, ok := p.read()
if !ok {
p.unread(len(toks) + 1)
return false
}
if tok.Equal(TOKEN_EOF, TOKEN_DOUBLE_NEWLINE) {
p.unread(len(toks) + 1)
return false
}
if tok.Equal(marker) {
break
}
toks = append(toks, tok)
}
if toks[len(toks)-1].Type == TYPE_WHITESPACE {
p.unread(len(toks) + 2)
return false
}
par := NewParserFromChannel(NewTokenStream(toks).Out())
par.Parse()
var b Element
switch marker.Type {
case TYPE_ASTERISK:
b = Strong(par.Elements())
case TYPE_UNDERSCORE:
b = Emphasis(par.Elements())
case TYPE_BACKTICK:
b = Code(par.Elements())
}
p.appendElement(b)
return true
}

101
styles_test.go
View File

@ -1,101 +0,0 @@
package adoc_test
import (
"strings"
"testing"
"ewintr.nl/adoc"
"ewintr.nl/go-kit/test"
)
func TestStyles(t *testing.T) {
for _, tc := range []struct {
name string
input string
exp []adoc.Element
}{
{
name: "strong",
input: "*strong*",
exp: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Strong{adoc.Word("strong")},
},
),
},
},
{
name: "emphasis",
input: "_emphasis_",
exp: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Emphasis{adoc.Word("emphasis")},
},
),
},
},
{
name: "code",
input: "`code`",
exp: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Code{adoc.Word("code")},
},
),
},
},
{
name: "mixed",
input: "some `code code` in plain",
exp: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Word("some"),
adoc.WhiteSpace(" "),
adoc.Code{
adoc.Word("code"),
adoc.WhiteSpace(" "),
adoc.Word("code"),
},
adoc.WhiteSpace(" "),
adoc.Word("in"),
adoc.WhiteSpace(" "),
adoc.Word("plain"),
},
),
},
},
{
name: "incomplete",
input: "a *word",
exp: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Word("a"),
adoc.WhiteSpace(" "),
adoc.Word("*"),
adoc.Word("word"),
}),
},
},
{
name: "trailing space",
input: "*word *",
exp: []adoc.Element{
adoc.Paragraph([]adoc.Element{
adoc.Word("*"),
adoc.Word("word"),
adoc.WhiteSpace(" "),
adoc.Word("*"),
}),
},
},
} {
t.Run(tc.name, func(t *testing.T) {
par := adoc.NewParser(strings.NewReader(tc.input))
exp := &adoc.ADoc{
Attributes: map[string]string{},
Content: tc.exp,
}
test.Equals(t, exp, par.Parse())
})
}
}

52
subtitle.go
View File

@ -1,52 +0,0 @@
package adoc
type SubTitle string
func (st SubTitle) Text() string { return string(st) }
type SubSubTitle string
func (st SubSubTitle) Text() string { return string(st) }
func (p *Parser) trySubTitle() bool {
toks, ok := p.readN(2)
if !ok {
return false
}
if toks[0].Type != TYPE_EQUALSIGN || toks[0].Len() < 2 || toks[1].Type != TYPE_WHITESPACE {
p.unread(2)
return false
}
for {
tok, ok := p.read()
if !ok {
p.unread(len(toks))
return false
}
if (tok.Type == TYPE_NEWLINE && tok.Len() > 1) || tok.Equal(TOKEN_EOF) {
break
}
toks = append(toks, tok)
}
var title string
for _, tok := range toks[2:] {
if tok.Type == TYPE_NEWLINE {
continue
}
title += p.makePlain(tok).Text()
}
switch toks[0].Len() {
case 2:
p.appendElement(SubTitle(title))
case 3:
p.appendElement(SubSubTitle(title))
default:
// ignore lower levels for now
p.unread(len(toks))
return false
}
return true
}

2
lexer.go → token/lexer.go
View File

@ -1,4 +1,4 @@
package adoc
package token
import (
"bufio"

40
lexer_test.go → token/lexer_test.go
View File

@ -1,32 +1,32 @@
package adoc_test
package token_test
import (
"strings"
"testing"
"time"
"ewintr.nl/adoc"
"ewintr.nl/adoc/token"
"ewintr.nl/go-kit/test"
)
func TestLexer(t *testing.T) {
word := adoc.TYPE_WORD
ws := adoc.TYPE_WHITESPACE
nl := adoc.TYPE_NEWLINE
eq := adoc.TYPE_EQUALSIGN
bt := adoc.TYPE_BACKTICK
as := adoc.TYPE_ASTERISK
un := adoc.TYPE_UNDERSCORE
word := token.TYPE_WORD
ws := token.TYPE_WHITESPACE
nl := token.TYPE_NEWLINE
eq := token.TYPE_EQUALSIGN
bt := token.TYPE_BACKTICK
as := token.TYPE_ASTERISK
un := token.TYPE_UNDERSCORE
for _, tc := range []struct {
name string
input string
exp []adoc.Token
exp []token.Token
}{
{
name: "word string",
input: "one two",
exp: []adoc.Token{
exp: []token.Token{
{Type: word, Literal: "one"},
{Type: ws, Literal: " "},
{Type: word, Literal: "two"},
@ -35,7 +35,7 @@ func TestLexer(t *testing.T) {
{
name: "punctuation",
input: `. ,`,
exp: []adoc.Token{
exp: []token.Token{
{Type: word, Literal: "."},
{Type: ws, Literal: " "},
{Type: word, Literal: ","},
@ -44,24 +44,24 @@ func TestLexer(t *testing.T) {
{
name: "whitespace",
input: " \t",
exp: []adoc.Token{
exp: []token.Token{
{Type: ws, Literal: " \t"},
},
},
{
name: "tab",
input: "\t",
exp: []adoc.Token{{Type: ws, Literal: "\t"}},
exp: []token.Token{{Type: ws, Literal: "\t"}},
},
{
name: "newlines",
input: "\n\n\n",
exp: []adoc.Token{{Type: nl, Literal: "\n\n\n"}},
exp: []token.Token{{Type: nl, Literal: "\n\n\n"}},
},
{
name: "special chars",
input: "=*_",
exp: []adoc.Token{
exp: []token.Token{
{Type: eq, Literal: "="},
{Type: as, Literal: "*"},
{Type: un, Literal: "_"},
@ -70,7 +70,7 @@ func TestLexer(t *testing.T) {
{
name: "mixed",
input: "This is a line with mixed \t `stuff`, see\t==?",
exp: []adoc.Token{
exp: []token.Token{
{Type: word, Literal: "This"},
{Type: ws, Literal: " "},
{Type: word, Literal: "is"},
@ -97,8 +97,8 @@ func TestLexer(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
input := strings.NewReader(tc.input)
lex := adoc.NewLexer(input)
act := []adoc.Token{}
lex := token.NewLexer(input)
act := []token.Token{}
stop := time.Now().Add(3 * time.Second)
T:
@ -118,7 +118,7 @@ func TestLexer(t *testing.T) {
}
test.OK(t, lex.Error())
exp := append(tc.exp, adoc.TOKEN_EOF)
exp := append(tc.exp, token.TOKEN_EOF)
test.Equals(t, exp, act)
})
}

2
token.go → token/token.go
View File

@ -1,4 +1,4 @@
package adoc
package token
const (
TYPE_EOF TokenType = iota

65
token/tokenreader.go Normal file
View File

@ -0,0 +1,65 @@
package token
type TokenReader struct {
in chan Token
ts []Token
r int
}
func NewTokenReader(in chan Token) *TokenReader {
return &TokenReader{
in: in,
ts: []Token{},
}
}
func (tr *TokenReader) Read(n int) ([]Token, bool) {
toks := []Token{}
for i := 0; i < n; i++ {
tok, ok := tr.readOne()
if !ok {
tr.Unread(len(toks))
return []Token{}, false
}
toks = append(toks, tok)
}
return toks, true
}
func (tr *TokenReader) Unread(n int) bool {
//if n > tr.r {
// return false
//}
for i := n; i > 0; i-- {
tr.r--
}
return true
}
func (tr *TokenReader) Discard() {
tr.ts = tr.ts[tr.r:]
tr.r = 0
}
func (tr *TokenReader) readOne() (Token, bool) {
if tr.r == len(tr.ts) {
if ok := tr.consume(); !ok {
return Token{}, false
}
}
tok := tr.ts[tr.r]
tr.r++
return tok, true
}
func (tr *TokenReader) consume() bool {
tok, ok := <-tr.in
if !ok {
return false
}
tr.ts = append(tr.ts, tok)
return true
}

3
tokenstream.go → token/tokenstream.go
View File

@ -1,4 +1,4 @@
package adoc
package token
type TokenStream struct {
ts []Token
@ -20,6 +20,7 @@ func (s *TokenStream) run() {
for _, tok := range s.ts {
s.out <- tok
}
s.out <- TOKEN_EOS
close(s.out)
}