decouple packages

adoc.go

@@ -2,6 +2,8 @@ package adoc
 
 import (
 	"time"
+
+	"ewintr.nl/adoc/element"
 )
 
 type ADoc struct {
@@ -10,12 +12,12 @@ type ADoc struct {
 	Author     string
 	Path       string
 	Date       time.Time
-	Content    []Element
+	Content    []element.Element
 }
 
 func NewADoc() *ADoc {
 	return &ADoc{
 		Attributes: map[string]string{},
-		Content:    []Element{},
+		Content:    []element.Element{},
 	}
 }

codeblock.go

@@ -1,43 +0,0 @@
-package adoc
-
-type CodeBlock []Element
-
-func (cb CodeBlock) Text() string {
-	txt := ""
-	for _, e := range cb {
-		txt += e.Text()
-	}
-
-	return txt
-}
-
-func (p *Parser) tryCodeBlock() bool {
-	delimiter := Token{Type: TYPE_DASH, Literal: "----"}
-	toks, ok := p.readN(2)
-	if !ok {
-		return false
-	}
-	if !toks[0].Equal(delimiter) || toks[1].Type != TYPE_NEWLINE {
-		p.unread(2)
-		return false
-	}
-	for {
-		tok, ok := p.read()
-		if !ok {
-			p.unread(len(toks))
-			return false
-		}
-		if tok.Equal(delimiter) {
-			break
-		}
-		toks = append(toks, tok)
-	}
-
-	cb := CodeBlock{}
-	for _, tok := range toks[2:] {
-		cb = append(cb, p.makePlain(tok))
-	}
-	p.appendElement(cb)
-
-	return true
-}

element.go

@@ -1,26 +0,0 @@
-package adoc
-
-type Element interface {
-	Text() string
-}
-
-type Empty struct{}
-
-func (e Empty) Text() string { return "" }
-
-type Word string
-
-func (w Word) Text() string { return string(w) }
-
-type WhiteSpace string
-
-func (ws WhiteSpace) Text() string { return string(ws) }
-
-type Image struct {
-	Src string
-	Alt string
-}
-
-func (i Image) Text() string {
-	return i.Alt
-}

element/codeblock.go

@@ -0,0 +1,54 @@
+package element
+
+import (
+	"ewintr.nl/adoc/token"
+)
+
+type CodeBlock []Element
+
+func (cb CodeBlock) Text() string {
+	txt := ""
+	for _, e := range cb {
+		txt += e.Text()
+	}
+
+	return txt
+}
+
+func (cb CodeBlock) Append(els []Element) Element {
+	return CodeBlock{append(cb, els...)}
+}
+
+func NewCodeBlockFromTokens(p ReadUnreader) (ParseResult, bool) {
+	delimiter := token.Token{Type: token.TYPE_DASH, Literal: "----"}
+	toks, ok := p.Read(2)
+	if !ok {
+		return ParseResult{}, false
+	}
+	if !toks[0].Equal(delimiter) || toks[1].Type != token.TYPE_NEWLINE {
+		p.Unread(2)
+		return ParseResult{}, false
+	}
+	for {
+		ntoks, ok := p.Read(1)
+		if !ok {
+			p.Unread(len(toks))
+			return ParseResult{}, false
+		}
+		tok := ntoks[0]
+		if tok.Equal(delimiter) {
+			break
+		}
+		toks = append(toks, tok)
+	}
+
+	cb := CodeBlock{}
+	for _, tok := range toks[2:] {
+		cb = append(cb, MakePlain(tok))
+	}
+
+	return ParseResult{
+		Element: cb,
+		Inner:   []token.Token{},
+	}, true
+}

element/codeblock_test.go

@@ -1,10 +1,12 @@
-package adoc_test
+package element_test
 
 import (
 	"strings"
 	"testing"
 
 	"ewintr.nl/adoc"
+	"ewintr.nl/adoc/element"
+	"ewintr.nl/adoc/parser"
 	"ewintr.nl/go-kit/test"
 )
 
@@ -20,7 +22,7 @@ func TestCodeBlock(t *testing.T) {
 ----`,
 			exp: &adoc.ADoc{
 				Attributes: map[string]string{},
-				Content:    []adoc.Element{adoc.CodeBlock{}},
+				Content:    []element.Element{element.CodeBlock{}},
 			},
 		},
 		{
@@ -32,11 +34,11 @@ more
 ----`,
 			exp: &adoc.ADoc{
 				Attributes: map[string]string{},
-				Content: []adoc.Element{adoc.CodeBlock{
+				Content: []element.Element{element.CodeBlock{
-					adoc.Word("code"),
+					element.Word("code"),
-					adoc.WhiteSpace("\n\n"),
+					element.WhiteSpace("\n\n"),
-					adoc.Word("more"),
+					element.Word("more"),
-					adoc.WhiteSpace("\n"),
+					element.WhiteSpace("\n"),
 				}},
 			},
 		},
@@ -49,22 +51,22 @@ more
 `,
 			exp: &adoc.ADoc{
 				Attributes: map[string]string{},
-				Content: []adoc.Element{
+				Content: []element.Element{
-					adoc.Paragraph{
+					element.Paragraph{[]element.Element{
-						adoc.Word("----"),
+						element.Word("----"),
-						adoc.WhiteSpace("\n"),
+						element.WhiteSpace("\n"),
-						adoc.Word("code"),
+						element.Word("code"),
-					},
+					}},
-					adoc.Paragraph{
+					element.Paragraph{[]element.Element{
-						adoc.Word("more"),
+						element.Word("more"),
-						adoc.WhiteSpace("\n"),
+						element.WhiteSpace("\n"),
-					},
+					}},
 				},
 			},
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			par := adoc.NewParser(strings.NewReader(tc.input))
+			par := parser.New(strings.NewReader(tc.input))
 			test.Equals(t, tc.exp, par.Parse())
 		})
 	}

element/element.go

@@ -0,0 +1,31 @@
+package element
+
+import "ewintr.nl/adoc/token"
+
+type Element interface {
+	Text() string
+	Append([]Element) Element
+}
+
+type ParseResult struct {
+	Element Element
+	Inner   []token.Token
+}
+
+type ReadUnreader interface {
+	Read(n int) ([]token.Token, bool)
+	Unread(n int) bool
+}
+
+type Empty struct{}
+
+func (e Empty) Text() string { return "" }
+
+type Image struct {
+	Src string
+	Alt string
+}
+
+func (i Image) Text() string {
+	return i.Alt
+}

element/header.go

@@ -0,0 +1,94 @@
+package element
+
+import (
+	"time"
+
+	"ewintr.nl/adoc/token"
+)
+
+type Header struct {
+	Title      string
+	Author     string
+	Date       time.Time
+	Attributes map[string]string
+}
+
+func (h Header) Text() string               { return h.Title }
+func (h Header) Append(_ []Element) Element { return h }
+
+func NewHeaderFromTokens(tr ReadUnreader) (ParseResult, bool) {
+	toks, ok := tr.Read(2)
+	if !ok {
+		return ParseResult{}, false
+	}
+	if !toks[0].Equal(token.Token{Type: token.TYPE_EQUALSIGN, Literal: "="}) {
+		tr.Unread(2)
+		return ParseResult{}, false
+	}
+	if toks[1].Type != token.TYPE_WHITESPACE {
+		tr.Unread(2)
+		return ParseResult{}, false
+	}
+	for {
+		ntoks, ok := tr.Read(1)
+		if !ok {
+			tr.Unread(len(toks))
+			return ParseResult{}, false
+		}
+		if ntoks[0].Equal(token.TOKEN_EOF, token.TOKEN_DOUBLE_NEWLINE) {
+			break
+		}
+		toks = append(toks, ntoks[0])
+	}
+
+	h := &Header{
+		Attributes: map[string]string{},
+	}
+	lines := token.Split(toks[2:], token.TYPE_NEWLINE)
+	h.Title = token.Literals(lines[0])
+
+	for _, line := range lines[1:] {
+		switch {
+		case tryHeaderDate(h, line):
+			continue
+		case tryHeaderField(h, line):
+			continue
+		default:
+			h.Author = token.Literals(line)
+		}
+	}
+
+	return ParseResult{
+		Element: *h,
+		Inner:   []token.Token{},
+	}, true
+}
+
+func tryHeaderField(h *Header, line []token.Token) bool {
+	if len(line) < 4 {
+		return false
+	}
+	pair := token.Split(line, token.TYPE_WHITESPACE)
+	if len(pair) != 2 {
+		return false
+	}
+	key, value := pair[0], pair[1]
+
+	if !token.HasPattern(key, []token.TokenType{token.TYPE_COLON, token.TYPE_WORD, token.TYPE_COLON}) {
+		return false
+	}
+
+	h.Attributes[key[1].Literal] = token.Literals(value)
+
+	return true
+}
+
+func tryHeaderDate(h *Header, line []token.Token) bool {
+	date, err := time.Parse("2006-01-02", token.Literals(line))
+	if err != nil {
+		return false
+	}
+	h.Date = date
+
+	return true
+}

element/header_test.go

@@ -1,4 +1,4 @@
-package adoc_test
+package element_test
 
 import (
 	"strings"
@@ -6,6 +6,8 @@ import (
 	"time"
 
 	"ewintr.nl/adoc"
+	"ewintr.nl/adoc/element"
+	"ewintr.nl/adoc/parser"
 	"ewintr.nl/go-kit/test"
 )
 
@@ -21,7 +23,7 @@ func TestHeader(t *testing.T) {
 			exp: &adoc.ADoc{
 				Title:      "Title",
 				Attributes: map[string]string{},
-				Content:    []adoc.Element{},
+				Content:    []element.Element{},
 			},
 		},
 		{
@@ -46,18 +48,18 @@ First paragraph`,
 					"key1": "value1",
 					"key2": "value2",
 				},
-				Content: []adoc.Element{
+				Content: []element.Element{
-					adoc.Paragraph([]adoc.Element{
+					element.Paragraph{[]element.Element{
-						adoc.Word("First"),
+						element.Word("First"),
-						adoc.WhiteSpace(" "),
+						element.WhiteSpace(" "),
-						adoc.Word("paragraph"),
+						element.Word("paragraph"),
-					}),
+					}},
 				},
 			},
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			par := adoc.NewParser(strings.NewReader(tc.input))
+			par := parser.New(strings.NewReader(tc.input))
 			test.Equals(t, tc.exp, par.Parse())
 		})
 	}

element/link.go

@@ -0,0 +1,72 @@
+package element
+
+import (
+	"ewintr.nl/adoc/token"
+)
+
+type Link struct {
+	URL   string
+	Title string
+}
+
+func (l Link) Text() string               { return l.Title }
+func (l Link) Append(_ []Element) Element { return l }
+
+func NewLinkFromTokens(tr ReadUnreader) (ParseResult, bool) {
+	tok, ok := tr.Read(1)
+	if !ok {
+		return ParseResult{}, false
+	}
+	tr.Unread(1)
+	if tok[0].Type != token.TYPE_WORD {
+		return ParseResult{}, false
+	}
+
+	url, title := []token.Token{}, []token.Token{}
+	lb := false
+	count := 0
+	for {
+		ntoks, ok := tr.Read(1)
+		if !ok {
+			tr.Unread(count)
+			return ParseResult{}, false
+		}
+		count++
+		tok := ntoks[0]
+		if tok.Equal(token.TOKEN_EOF, token.TOKEN_EOS, token.TOKEN_NEWLINE, token.TOKEN_DOUBLE_NEWLINE) {
+			tr.Unread(count)
+			return ParseResult{}, false
+		}
+		if !lb && tok.Type == token.TYPE_WHITESPACE {
+			tr.Unread(count)
+			return ParseResult{}, false
+		}
+		if tok.Equal(token.Token{Type: token.TYPE_BRACKETOPEN, Literal: "["}) {
+			lb = true
+			continue
+		}
+		if lb && tok.Equal(token.Token{Type: token.TYPE_BRACKETCLOSE, Literal: "]"}) {
+			break
+		}
+
+		if lb {
+			title = append(title, tok)
+			continue
+		}
+		url = append(url, tok)
+	}
+
+	if len(url) == 0 || !lb {
+		tr.Unread(count)
+		return ParseResult{}, false
+	}
+	link := Link{
+		URL:   token.Literals(url),
+		Title: token.Literals(title),
+	}
+
+	return ParseResult{
+		Element: link,
+		Inner:   []token.Token{},
+	}, true
+}

element/link_test.go

@@ -1,10 +1,12 @@
-package adoc_test
+package element_test
 
 import (
 	"strings"
 	"testing"
 
 	"ewintr.nl/adoc"
+	"ewintr.nl/adoc/element"
+	"ewintr.nl/adoc/parser"
 	"ewintr.nl/go-kit/test"
 )
 
@@ -12,41 +14,41 @@ func TestLink(t *testing.T) {
 	for _, tc := range []struct {
 		name  string
 		input string
-		exp   []adoc.Element
+		exp   []element.Element
 	}{
 		{
 			name:  "simple",
 			input: "a link[title] somewhere",
-			exp: []adoc.Element{
+			exp: []element.Element{
-				adoc.Paragraph([]adoc.Element{
+				element.Paragraph{Elements: []element.Element{
-					adoc.Word("a"),
+					element.Word("a"),
-					adoc.WhiteSpace(" "),
+					element.WhiteSpace(" "),
-					adoc.Link{
+					element.Link{
 						URL:   "link",
 						Title: "title",
 					},
-					adoc.WhiteSpace(" "),
+					element.WhiteSpace(" "),
-					adoc.Word("somewhere"),
+					element.Word("somewhere"),
-				}),
+				}},
 			},
 		},
 		{
 			name:  "with underscore",
 			input: "check https://example.com/some_url[some url]",
-			exp: []adoc.Element{
+			exp: []element.Element{
-				adoc.Paragraph([]adoc.Element{
+				element.Paragraph{Elements: []element.Element{
-					adoc.Word("check"),
+					element.Word("check"),
-					adoc.WhiteSpace(" "),
+					element.WhiteSpace(" "),
-					adoc.Link{
+					element.Link{
 						URL:   "https://example.com/some_url",
 						Title: "some url",
 					},
-				}),
+				}},
 			},
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			par := adoc.NewParser(strings.NewReader(tc.input))
+			par := parser.New(strings.NewReader(tc.input))
 			exp := &adoc.ADoc{
 				Attributes: map[string]string{},
 				Content:    tc.exp,

element/list.go

@@ -0,0 +1,56 @@
+package element
+
+import (
+	"ewintr.nl/adoc/token"
+)
+
+type List []ListItem
+
+func (l List) Text() string {
+	txt := ""
+	for _, li := range l {
+		txt += li.Text()
+	}
+
+	return txt
+}
+
+func (l List) Append(els []Element) Element {
+	for _, el := range els {
+		if val, ok := el.(ListItem); ok {
+			l = append(l, val)
+		}
+	}
+	return l
+}
+
+func NewListFromTokens(tr ReadUnreader) (ParseResult, bool) {
+	toks, ok := tr.Read(2)
+	if !ok {
+		return ParseResult{}, false
+	}
+	if !toks[0].Equal(token.TOKEN_ASTERISK) || toks[1].Type != token.TYPE_WHITESPACE {
+		tr.Unread(2)
+		return ParseResult{}, false
+	}
+
+	tr.Unread(2)
+	toks = []token.Token{}
+	for {
+		ntoks, ok := tr.Read(1)
+		if !ok {
+			tr.Unread(len(toks))
+			return ParseResult{}, false
+		}
+		tok := ntoks[0]
+		if tok.Equal(token.TOKEN_DOUBLE_NEWLINE, token.TOKEN_EOF) {
+			break
+		}
+		toks = append(toks, tok)
+	}
+
+	return ParseResult{
+		Element: List{},
+		Inner:   toks,
+	}, true
+}

element/list_test.go

@@ -0,0 +1,79 @@
+package element_test
+
+import (
+	"strings"
+	"testing"
+
+	"ewintr.nl/adoc"
+	"ewintr.nl/adoc/element"
+	"ewintr.nl/adoc/parser"
+	"ewintr.nl/go-kit/test"
+)
+
+func TestList(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		input string
+		exp   []element.Element
+	}{
+		{
+			name:  "one item",
+			input: `* item 1`,
+			exp: []element.Element{
+				element.List([]element.ListItem{
+					{element.Word("item"), element.WhiteSpace(" "), element.Word("1")},
+				},
+				)},
+		},
+		{
+			name: "multiple",
+			input: `* item 1
+* item 2
+* item 3`,
+			exp: []element.Element{
+				element.List([]element.ListItem{
+					{element.Word("item"), element.WhiteSpace(" "), element.Word("1")},
+					{element.Word("item"), element.WhiteSpace(" "), element.Word("2")},
+					{element.Word("item"), element.WhiteSpace(" "), element.Word("3")},
+				})},
+		},
+		{
+			name: "double with pararaph",
+			input: `* item 1
+
+* item 2
+* item 3
+
+and some text`,
+			exp: []element.Element{
+				element.List(
+					[]element.ListItem{
+						{element.Word("item"), element.WhiteSpace(" "), element.Word("1")},
+					},
+				),
+				element.List(
+					[]element.ListItem{
+						{element.Word("item"), element.WhiteSpace(" "), element.Word("2")},
+						{element.Word("item"), element.WhiteSpace(" "), element.Word("3")},
+					},
+				),
+				element.Paragraph{Elements: []element.Element{
+					element.Word("and"),
+					element.WhiteSpace(" "),
+					element.Word("some"),
+					element.WhiteSpace(" "),
+					element.Word("text"),
+				}},
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			par := parser.New(strings.NewReader(tc.input))
+			exp := &adoc.ADoc{
+				Attributes: map[string]string{},
+				Content:    tc.exp,
+			}
+			test.Equals(t, exp, par.Parse())
+		})
+	}
+}

element/listitem.go

@@ -0,0 +1,54 @@
+package element
+
+import (
+	"ewintr.nl/adoc/token"
+)
+
+type ListItem []Element
+
+func (li ListItem) Text() string {
+	txt := ""
+	for _, e := range li {
+		txt += e.Text()
+	}
+
+	return txt
+}
+
+func (li ListItem) Append(els []Element) Element {
+	for _, el := range els {
+		li = append(li, el)
+	}
+
+	return li
+}
+
+func NewListItemFromTokens(tr ReadUnreader) (ParseResult, bool) {
+	toks, ok := tr.Read(2)
+	if !ok {
+		return ParseResult{}, false
+	}
+	if !toks[0].Equal(token.TOKEN_ASTERISK) || toks[1].Type != token.TYPE_WHITESPACE {
+		tr.Unread(2)
+		return ParseResult{}, false
+	}
+
+	toks = []token.Token{}
+	for {
+		ntoks, ok := tr.Read(1)
+		if !ok {
+			tr.Unread(len(toks))
+			return ParseResult{}, false
+		}
+		tok := ntoks[0]
+		if tok.Equal(token.TOKEN_NEWLINE, token.TOKEN_EOS) {
+			break
+		}
+		toks = append(toks, tok)
+	}
+
+	return ParseResult{
+		Element: ListItem{},
+		Inner:   toks,
+	}, true
+}

element/paragraph.go

@@ -0,0 +1,48 @@
+package element
+
+import (
+	"ewintr.nl/adoc/token"
+)
+
+type Paragraph struct {
+	Elements []Element
+}
+
+func (p Paragraph) Text() string {
+	txt := ""
+	for _, el := range p.Elements {
+		txt += el.Text()
+	}
+
+	return txt
+}
+
+func (p Paragraph) Append(els []Element) Element {
+	return Paragraph{
+		Elements: append(p.Elements, els...),
+	}
+}
+
+func NewParagraphFromTokens(tr ReadUnreader) (ParseResult, bool) {
+	toks := []token.Token{}
+	for {
+		tok, ok := tr.Read(1)
+		if !ok {
+			tr.Unread(len(toks))
+			return ParseResult{}, false
+		}
+		if tok[0].Equal(token.TOKEN_EOF, token.TOKEN_DOUBLE_NEWLINE) {
+			break
+		}
+		toks = append(toks, tok[0])
+	}
+
+	if len(toks) == 0 {
+		return ParseResult{}, false
+	}
+
+	return ParseResult{
+		Element: Paragraph{Elements: []Element{}},
+		Inner:   toks,
+	}, true
+}

element/paragraph_test.go

@@ -0,0 +1,78 @@
+package element_test
+
+import (
+	"strings"
+	"testing"
+
+	"ewintr.nl/adoc"
+	"ewintr.nl/adoc/element"
+	"ewintr.nl/adoc/parser"
+	"ewintr.nl/go-kit/test"
+)
+
+func TestParagraph(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		input string
+		exp   *adoc.ADoc
+	}{
+		{
+			name:  "single paragraph",
+			input: "some text",
+			exp: &adoc.ADoc{
+				Attributes: map[string]string{},
+				Content: []element.Element{
+					element.Paragraph{Elements: []element.Element{
+						element.Word("some"),
+						element.WhiteSpace(" "),
+						element.Word("text"),
+					}},
+				}},
+		},
+		{
+			name: "title with paragraphs",
+			input: `= Title
+
+paragraph one
+
+paragraph two`,
+			exp: &adoc.ADoc{
+				Title:      "Title",
+				Attributes: map[string]string{},
+				Content: []element.Element{
+					element.Paragraph{Elements: []element.Element{
+						element.Word("paragraph"),
+						element.WhiteSpace(" "),
+						element.Word("one"),
+					}},
+					element.Paragraph{Elements: []element.Element{
+						element.Word("paragraph"),
+						element.WhiteSpace(" "),
+						element.Word("two"),
+					}},
+				},
+			},
+		},
+		{
+			name: "three with trailing newline",
+			input: `one
+
+two
+
+three
+`,
+			exp: &adoc.ADoc{
+				Attributes: map[string]string{},
+				Content: []element.Element{
+					element.Paragraph{Elements: []element.Element{element.Word("one")}},
+					element.Paragraph{Elements: []element.Element{element.Word("two")}},
+					element.Paragraph{Elements: []element.Element{element.Word("three"), element.WhiteSpace("\n")}},
+				}},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			par := parser.New(strings.NewReader(tc.input))
+			test.Equals(t, tc.exp, par.Parse())
+		})
+	}
+}

element/plain.go

@@ -0,0 +1,25 @@
+package element
+
+import "ewintr.nl/adoc/token"
+
+type Word string
+
+func (w Word) Text() string               { return string(w) }
+func (w Word) Append(_ []Element) Element { return w }
+
+type WhiteSpace string
+
+func (ws WhiteSpace) Text() string               { return string(ws) }
+func (ws WhiteSpace) Append(_ []Element) Element { return ws }
+
+func MakePlain(tok token.Token) Element {
+	switch tok.Type {
+	case token.TYPE_WHITESPACE:
+		return WhiteSpace(tok.Literal)
+	case token.TYPE_NEWLINE:
+		return WhiteSpace(tok.Literal)
+	default:
+		return Word(tok.Literal)
+	}
+
+}

element/styles.go

@@ -0,0 +1,106 @@
+package element
+
+import "ewintr.nl/adoc/token"
+
+type Strong []Element
+
+func (st Strong) Text() string {
+	var txt string
+	for _, e := range st {
+		txt += e.Text()
+	}
+
+	return txt
+}
+
+func (st Strong) Append(els []Element) Element {
+	return append(st, els...)
+}
+
+type Emphasis []Element
+
+func (em Emphasis) Text() string {
+	var txt string
+	for _, e := range em {
+		txt += e.Text()
+	}
+
+	return txt
+}
+
+func (em Emphasis) Append(els []Element) Element {
+	return append(em, els...)
+}
+
+type Code []Element
+
+func (c Code) Text() string {
+	var txt string
+	for _, e := range c {
+		txt += e.Text()
+	}
+
+	return txt
+}
+
+func (c Code) Append(els []Element) Element {
+	return append(c, els...)
+}
+
+func NewStyleFromTokens(tr ReadUnreader) (ParseResult, bool) {
+	toks, ok := tr.Read(2)
+	if !ok {
+		return ParseResult{}, false
+	}
+	markers := []token.Token{
+		{Type: token.TYPE_ASTERISK, Literal: "*"},
+		{Type: token.TYPE_UNDERSCORE, Literal: "_"},
+		{Type: token.TYPE_BACKTICK, Literal: "`"},
+	}
+	if !toks[0].Equal(markers...) {
+		tr.Unread(2)
+		return ParseResult{}, false
+	}
+	if toks[1].Type == token.TYPE_WHITESPACE {
+		tr.Unread(2)
+		return ParseResult{}, false
+	}
+
+	marker := toks[0]
+	toks = []token.Token{toks[1]}
+	for {
+		ntoks, ok := tr.Read(1)
+		if !ok {
+			tr.Unread(len(toks) + 1)
+			return ParseResult{}, false
+		}
+		tok := ntoks[0]
+		if tok.Equal(token.TOKEN_EOF, token.TOKEN_DOUBLE_NEWLINE) {
+			tr.Unread(len(toks) + 1)
+			return ParseResult{}, false
+		}
+		if tok.Equal(marker) {
+			break
+		}
+		toks = append(toks, tok)
+	}
+	if toks[len(toks)-1].Type == token.TYPE_WHITESPACE {
+		tr.Unread(len(toks) + 2)
+		return ParseResult{}, false
+	}
+
+	var s Element
+	switch marker.Type {
+	case token.TYPE_ASTERISK:
+		s = Strong{}
+	case token.TYPE_UNDERSCORE:
+		s = Emphasis{}
+	case token.TYPE_BACKTICK:
+		s = Code{}
+	}
+
+	return ParseResult{
+		Element: s,
+		Inner:   toks,
+	}, true
+}

element/styles_test.go

@@ -0,0 +1,103 @@
+package element_test
+
+import (
+	"strings"
+	"testing"
+
+	"ewintr.nl/adoc"
+	"ewintr.nl/adoc/element"
+	"ewintr.nl/adoc/parser"
+	"ewintr.nl/go-kit/test"
+)
+
+func TestStyles(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		input string
+		exp   []element.Element
+	}{
+		{
+			name:  "strong",
+			input: "*strong*",
+			exp: []element.Element{
+				element.Paragraph{Elements: []element.Element{
+					element.Strong{element.Word("strong")},
+				},
+				},
+			},
+		},
+		{
+			name:  "emphasis",
+			input: "_emphasis_",
+			exp: []element.Element{
+				element.Paragraph{Elements: []element.Element{
+					element.Emphasis{element.Word("emphasis")},
+				},
+				},
+			},
+		},
+		{
+			name:  "code",
+			input: "`code`",
+			exp: []element.Element{
+				element.Paragraph{Elements: []element.Element{
+					element.Code{element.Word("code")},
+				},
+				},
+			},
+		},
+		{
+			name:  "mixed",
+			input: "some `code code` in plain",
+			exp: []element.Element{
+				element.Paragraph{Elements: []element.Element{
+					element.Word("some"),
+					element.WhiteSpace(" "),
+					element.Code{
+						element.Word("code"),
+						element.WhiteSpace(" "),
+						element.Word("code"),
+					},
+					element.WhiteSpace(" "),
+					element.Word("in"),
+					element.WhiteSpace(" "),
+					element.Word("plain"),
+				},
+				},
+			},
+		},
+		{
+			name:  "incomplete",
+			input: "a *word",
+			exp: []element.Element{
+				element.Paragraph{Elements: []element.Element{
+					element.Word("a"),
+					element.WhiteSpace(" "),
+					element.Word("*"),
+					element.Word("word"),
+				}},
+			},
+		},
+		{
+			name:  "trailing space",
+			input: "*word *",
+			exp: []element.Element{
+				element.Paragraph{Elements: []element.Element{
+					element.Word("*"),
+					element.Word("word"),
+					element.WhiteSpace(" "),
+					element.Word("*"),
+				}},
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			par := parser.New(strings.NewReader(tc.input))
+			exp := &adoc.ADoc{
+				Attributes: map[string]string{},
+				Content:    tc.exp,
+			}
+			test.Equals(t, exp, par.Parse())
+		})
+	}
+}

element/subtitle.go

@@ -0,0 +1,62 @@
+package element
+
+import "ewintr.nl/adoc/token"
+
+type SubTitle string
+
+func (st SubTitle) Text() string               { return string(st) }
+func (st SubTitle) Append(_ []Element) Element { return st }
+
+type SubSubTitle string
+
+func (st SubSubTitle) Text() string               { return string(st) }
+func (st SubSubTitle) Append(_ []Element) Element { return st }
+
+func NewSubTitleFromTokens(tr ReadUnreader) (ParseResult, bool) {
+	toks, ok := tr.Read(2)
+	if !ok {
+		return ParseResult{}, false
+	}
+	if toks[0].Type != token.TYPE_EQUALSIGN || toks[0].Len() < 2 || toks[1].Type != token.TYPE_WHITESPACE {
+		tr.Unread(2)
+		return ParseResult{}, false
+	}
+
+	for {
+		ntoks, ok := tr.Read(1)
+		if !ok {
+			tr.Unread(len(toks))
+			return ParseResult{}, false
+		}
+		tok := ntoks[0]
+		if (tok.Type == token.TYPE_NEWLINE && tok.Len() > 1) || tok.Equal(token.TOKEN_EOF) {
+			break
+		}
+		toks = append(toks, tok)
+	}
+
+	var title string
+	for _, tok := range toks[2:] {
+		if tok.Type == token.TYPE_NEWLINE {
+			continue
+		}
+		title += MakePlain(tok).Text()
+	}
+
+	var el Element
+	switch toks[0].Len() {
+	case 2:
+		el = SubTitle(title)
+	case 3:
+		el = SubSubTitle(title)
+	default:
+		// ignore lower levels for now
+		tr.Unread(len(toks))
+		return ParseResult{}, false
+	}
+
+	return ParseResult{
+		Element: el,
+		Inner:   []token.Token{},
+	}, true
+}

element/subtitle_test.go

@@ -1,10 +1,12 @@
-package adoc_test
+package element_test
 
 import (
 	"strings"
 	"testing"
 
 	"ewintr.nl/adoc"
+	"ewintr.nl/adoc/element"
+	"ewintr.nl/adoc/parser"
 	"ewintr.nl/go-kit/test"
 )
 
@@ -12,27 +14,27 @@ func TestSubTitle(t *testing.T) {
 	for _, tc := range []struct {
 		name  string
 		input string
-		exp   []adoc.Element
+		exp   []element.Element
 	}{
 		{
 			name:  "empty",
 			input: "== ",
-			exp:   []adoc.Element{adoc.SubTitle("")},
+			exp:   []element.Element{element.SubTitle("")},
 		},
 		{
 			name:  "subtitle",
 			input: "== title with words",
-			exp:   []adoc.Element{adoc.SubTitle("title with words")},
+			exp:   []element.Element{element.SubTitle("title with words")},
 		},
 		{
 			name:  "subsubtitle",
 			input: "=== title",
-			exp:   []adoc.Element{adoc.SubSubTitle("title")},
+			exp:   []element.Element{element.SubSubTitle("title")},
 		},
 		{
 			name:  "trailing newline",
 			input: "== title\n",
-			exp:   []adoc.Element{adoc.SubTitle("title")},
+			exp:   []element.Element{element.SubTitle("title")},
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
@@ -40,7 +42,7 @@ func TestSubTitle(t *testing.T) {
 				Attributes: map[string]string{},
 				Content:    tc.exp,
 			}
-			par := adoc.NewParser(strings.NewReader(tc.input))
+			par := parser.New(strings.NewReader(tc.input))
 			test.Equals(t, exp, par.Parse())
 		})
 	}

header.go

@@ -1,76 +0,0 @@
-package adoc
-
-import (
-	"time"
-)
-
-func (p *Parser) tryHeader() bool {
-	toks, ok := p.readN(2)
-	if !ok {
-		return false
-	}
-	if !toks[0].Equal(Token{Type: TYPE_EQUALSIGN, Literal: "="}) {
-		p.unread(2)
-		return false
-	}
-	if toks[1].Type != TYPE_WHITESPACE {
-		p.unread(2)
-		return false
-	}
-	for {
-		tok, ok := p.read()
-		if !ok {
-			p.unread(len(toks))
-			return false
-		}
-		if tok.Equal(TOKEN_EOF, TOKEN_DOUBLE_NEWLINE) {
-			break
-		}
-		toks = append(toks, tok)
-	}
-
-	lines := Split(toks[2:], TYPE_NEWLINE)
-	p.doc.Title = Literals(lines[0])
-
-	for _, line := range lines[1:] {
-		switch {
-		case p.tryHeaderDate(line):
-			continue
-		case p.tryHeaderField(line):
-			continue
-		default:
-			p.doc.Author = Literals(line)
-		}
-	}
-
-	return true
-}
-
-func (p *Parser) tryHeaderField(line []Token) bool {
-	if len(line) < 4 {
-		return false
-	}
-	pair := Split(line, TYPE_WHITESPACE)
-	if len(pair) != 2 {
-		return false
-	}
-	key, value := pair[0], pair[1]
-
-	if !HasPattern(key, []TokenType{TYPE_COLON, TYPE_WORD, TYPE_COLON}) {
-		return false
-	}
-
-	p.doc.Attributes[key[1].Literal] = Literals(value)
-
-	return true
-}
-
-func (p *Parser) tryHeaderDate(line []Token) bool {
-	date, err := time.Parse("2006-01-02", Literals(line))
-	if err != nil {
-		return false
-	}
-	p.doc.Date = date
-
-	return true
-}

link.go

@@ -1,65 +0,0 @@
-package adoc
-
-import "fmt"
-
-type Link struct {
-	URL   string
-	Title string
-}
-
-func (l Link) Text() string { return l.Title }
-
-func (p *Parser) tryLink() bool {
-	tok, ok := p.peek()
-	if !ok {
-		return false
-	}
-	if tok.Type != TYPE_WORD {
-		return false
-	}
-
-	url, title := []Token{}, []Token{}
-	lb := false
-	count := 0
-	for {
-		tok, ok := p.read()
-		if !ok {
-			p.unread(count)
-			return false
-		}
-		count++
-		if tok.Equal(TOKEN_EOF, TOKEN_NEWLINE, TOKEN_DOUBLE_NEWLINE) {
-			p.unread(count)
-			return false
-		}
-		if !lb && tok.Type == TYPE_WHITESPACE {
-			p.unread(count)
-			return false
-		}
-		if tok.Equal(Token{Type: TYPE_BRACKETOPEN, Literal: "["}) {
-			lb = true
-			continue
-		}
-		if tok.Equal(Token{Type: TYPE_BRACKETCLOSE, Literal: "]"}) {
-			break
-		}
-
-		if lb {
-			title = append(title, tok)
-			continue
-		}
-		url = append(url, tok)
-	}
-	fmt.Printf("url: %+v, title: %+v\n", url, title)
-
-	if len(url) == 0 || !lb {
-		p.unread(count)
-		return false
-	}
-	link := Link{
-		URL:   Literals(url),
-		Title: Literals(title),
-	}
-	p.appendElement(link)
-	return true
-}

list.go

@@ -1,77 +0,0 @@
-package adoc
-
-type ListItem []Element
-
-func (li ListItem) Text() string {
-	txt := ""
-	for _, e := range li {
-		txt += e.Text()
-	}
-
-	return txt
-}
-
-type List []ListItem
-
-func (l List) Text() string {
-	txt := ""
-	for _, li := range l {
-		txt += li.Text()
-	}
-
-	return txt
-}
-
-func (p *Parser) tryList() bool {
-	toks, ok := p.readN(2)
-	if !ok {
-		return false
-	}
-	if !toks[0].Equal(TOKEN_ASTERISK) || toks[1].Type != TYPE_WHITESPACE {
-		p.unread(2)
-		return false
-	}
-
-	p.unread(2)
-	toks = []Token{}
-	toksCount := 0
-	var items []ListItem
-	for {
-		tok, ok := p.read()
-		if !ok {
-			p.unread(toksCount)
-			return false
-		}
-		if tok.Equal(TOKEN_NEWLINE, TOKEN_DOUBLE_NEWLINE, TOKEN_EOF) {
-			item, ok := tryListItem(toks)
-			if !ok {
-				p.unread(len(toks))
-				return false
-			}
-			items = append(items, item)
-			toks = []Token{}
-			if tok.Equal(TOKEN_DOUBLE_NEWLINE, TOKEN_EOF) {
-				break
-			}
-			continue
-		}
-		toks = append(toks, tok)
-		toksCount++
-	}
-
-	p.appendElement(List(items))
-
-	return true
-}
-
-func tryListItem(toks []Token) (ListItem, bool) {
-	if !toks[0].Equal(TOKEN_ASTERISK) || toks[1].Type != TYPE_WHITESPACE {
-		return ListItem{}, false
-	}
-	stream := NewTokenStream(toks[2:]).Out()
-	par := NewParserFromChannel(stream)
-	par.Parse()
-	b := ListItem(par.Elements())
-
-	return b, true
-}

list_test.go

@@ -1,77 +0,0 @@
-package adoc_test
-
-import (
-	"strings"
-	"testing"
-
-	"ewintr.nl/adoc"
-	"ewintr.nl/go-kit/test"
-)
-
-func TestList(t *testing.T) {
-	for _, tc := range []struct {
-		name  string
-		input string
-		exp   []adoc.Element
-	}{
-		{
-			name:  "one item",
-			input: `* item 1`,
-			exp: []adoc.Element{
-				adoc.List([]adoc.ListItem{
-					{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("1")},
-				},
-				)},
-		},
-		{
-			name: "multiple",
-			input: `* item 1
-* item 2
-* item 3`,
-			exp: []adoc.Element{
-				adoc.List([]adoc.ListItem{
-					{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("1")},
-					{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("2")},
-					{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("3")},
-				})},
-		},
-		{
-			name: "double with pararaph",
-			input: `* item 1
-
-* item 2
-* item 3
-
-and some text`,
-			exp: []adoc.Element{
-				adoc.List(
-					[]adoc.ListItem{
-						{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("1")},
-					},
-				),
-				adoc.List(
-					[]adoc.ListItem{
-						{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("2")},
-						{adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("3")},
-					},
-				),
-				adoc.Paragraph([]adoc.Element{
-					adoc.Word("and"),
-					adoc.WhiteSpace(" "),
-					adoc.Word("some"),
-					adoc.WhiteSpace(" "),
-					adoc.Word("text"),
-				}),
-			},
-		},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			par := adoc.NewParser(strings.NewReader(tc.input))
-			exp := &adoc.ADoc{
-				Attributes: map[string]string{},
-				Content:    tc.exp,
-			}
-			test.Equals(t, exp, par.Parse())
-		})
-	}
-}

paragraph.go

@@ -1,39 +0,0 @@
-package adoc
-
-type Paragraph []Element
-
-func (p Paragraph) Text() string {
-	txt := ""
-	for _, e := range p {
-		txt += e.Text()
-	}
-
-	return txt
-}
-
-func (p *Parser) tryParagraph() bool {
-	toks := []Token{}
-	for {
-		tok, ok := p.read()
-		if !ok {
-			p.unread(len(toks))
-			return false
-		}
-		if tok.Equal(TOKEN_EOF, TOKEN_DOUBLE_NEWLINE) {
-			break
-		}
-		toks = append(toks, tok)
-	}
-
-	if len(toks) == 0 {
-		return false
-	}
-
-	par := NewParserFromChannel(NewTokenStream(toks).Out())
-	par.Parse()
-	b := Paragraph(par.Elements())
-
-	p.appendElement(b)
-
-	return true
-}

paragraph_test.go

@@ -1,76 +0,0 @@
-package adoc_test
-
-import (
-	"strings"
-	"testing"
-
-	"ewintr.nl/adoc"
-	"ewintr.nl/go-kit/test"
-)
-
-func TestParagraph(t *testing.T) {
-	for _, tc := range []struct {
-		name  string
-		input string
-		exp   *adoc.ADoc
-	}{
-		{
-			name:  "single paragraph",
-			input: "some text",
-			exp: &adoc.ADoc{
-				Attributes: map[string]string{},
-				Content: []adoc.Element{
-					adoc.Paragraph([]adoc.Element{
-						adoc.Word("some"),
-						adoc.WhiteSpace(" "),
-						adoc.Word("text"),
-					}),
-				}},
-		},
-		{
-			name: "title with paragraphs",
-			input: `= Title
-
-paragraph one
-
-paragraph two`,
-			exp: &adoc.ADoc{
-				Title:      "Title",
-				Attributes: map[string]string{},
-				Content: []adoc.Element{
-					adoc.Paragraph([]adoc.Element{
-						adoc.Word("paragraph"),
-						adoc.WhiteSpace(" "),
-						adoc.Word("one"),
-					}),
-					adoc.Paragraph([]adoc.Element{
-						adoc.Word("paragraph"),
-						adoc.WhiteSpace(" "),
-						adoc.Word("two"),
-					}),
-				},
-			},
-		},
-		{
-			name: "three with trailing newline",
-			input: `one
-
-two
-
-three
-`,
-			exp: &adoc.ADoc{
-				Attributes: map[string]string{},
-				Content: []adoc.Element{
-					adoc.Paragraph([]adoc.Element{adoc.Word("one")}),
-					adoc.Paragraph([]adoc.Element{adoc.Word("two")}),
-					adoc.Paragraph([]adoc.Element{adoc.Word("three"), adoc.WhiteSpace("\n")}),
-				}},
-		},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			par := adoc.NewParser(strings.NewReader(tc.input))
-			test.Equals(t, tc.exp, par.Parse())
-		})
-	}
-}

parser.go

@@ -1,145 +0,0 @@
-package adoc
-
-import "io"
-
-type Parser struct {
-	in  chan Token
-	doc *ADoc
-	ts  []Token
-	r   int
-	els []Element
-}
-
-func NewParser(reader io.Reader) *Parser {
-	lex := NewLexer(reader)
-	return NewParserFromChannel(lex.Out())
-}
-
-func NewParserFromChannel(toks chan Token) *Parser {
-	return &Parser{
-		in:  toks,
-		doc: NewADoc(),
-		ts:  []Token{},
-		els: []Element{},
-	}
-}
-
-func (p *Parser) Parse() *ADoc {
-	p.tryHeader()
-
-	for {
-		if done := p.scan(); done {
-			break
-		}
-		p.discard()
-	}
-
-	p.doc.Content = p.els
-	return p.doc
-}
-
-func (p *Parser) Elements() []Element {
-	return p.els
-}
-
-func (p *Parser) scan() bool {
-	if _, ok := p.peek(); !ok {
-		return true
-	}
-
-	type tryFunc func() bool
-	for _, tf := range []tryFunc{
-		p.tryCodeBlock, p.trySubTitle, p.tryList, p.tryParagraph,
-		p.tryStyles, p.tryLink,
-	} {
-		if tf() {
-			return false
-		}
-	}
-
-	p.doPlain()
-
-	return false
-}
-
-func (p *Parser) doPlain() {
-	tok, ok := p.read()
-	if !ok {
-		return
-	}
-	p.appendElement(p.makePlain(tok))
-}
-
-func (p *Parser) makePlain(tok Token) Element {
-	switch tok.Type {
-	case TYPE_WHITESPACE:
-		return WhiteSpace(tok.Literal)
-	case TYPE_NEWLINE:
-		return WhiteSpace(tok.Literal)
-	default:
-		return Word(tok.Literal)
-	}
-
-}
-
-func (p *Parser) appendElement(b Element) {
-	p.els = append(p.els, b)
-}
-
-func (p *Parser) consume() bool {
-	tok, ok := <-p.in
-	if !ok {
-		return false
-	}
-	p.ts = append(p.ts, tok)
-
-	return true
-}
-
-func (p *Parser) peek() (Token, bool) {
-	if p.r == len(p.ts) {
-		if ok := p.consume(); !ok {
-			return Token{}, false
-		}
-	}
-
-	return p.ts[p.r], true
-}
-
-func (p *Parser) readN(count int) ([]Token, bool) {
-	toks := []Token{}
-	for i := 0; i < count; i++ {
-		tok, ok := p.read()
-		if !ok {
-			p.unread(len(toks))
-			return []Token{}, false
-		}
-		toks = append(toks, tok)
-	}
-
-	return toks, true
-}
-
-func (p *Parser) read() (Token, bool) {
-	if p.r == len(p.ts) {
-		if ok := p.consume(); !ok {
-			return Token{}, false
-		}
-	}
-
-	tok := p.ts[p.r]
-	p.r++
-
-	return tok, true
-}
-
-func (p *Parser) unread(count int) {
-	for i := count; i > 0; i-- {
-		p.r--
-	}
-}
-
-func (p *Parser) discard() {
-	p.ts = p.ts[p.r:]
-	p.r = 0
-}

parser/parser.go

@@ -0,0 +1,101 @@
+package parser
+
+import (
+	"io"
+
+	"ewintr.nl/adoc"
+	"ewintr.nl/adoc/element"
+	"ewintr.nl/adoc/token"
+)
+
+type Parser struct {
+	doc *adoc.ADoc
+	tr  *token.TokenReader
+	els []element.Element
+}
+
+func New(reader io.Reader) *Parser {
+	lex := token.NewLexer(reader)
+	return NewParserFromChannel(lex.Out())
+}
+
+func NewParserFromChannel(toks chan token.Token) *Parser {
+	return &Parser{
+		doc: adoc.NewADoc(),
+		tr:  token.NewTokenReader(toks),
+		els: []element.Element{},
+	}
+}
+
+func (p *Parser) Parse() *adoc.ADoc {
+	result, ok := element.NewHeaderFromTokens(p.tr)
+	if ok {
+		if h, ok := result.Element.(element.Header); ok {
+			p.doc.Title = h.Title
+			p.doc.Author = h.Author
+			p.doc.Date = h.Date
+			p.doc.Attributes = h.Attributes
+		}
+	}
+
+	for {
+		if done := p.scan(); done {
+			break
+		}
+		p.tr.Discard()
+	}
+
+	p.doc.Content = p.els
+	return p.doc
+}
+
+func (p *Parser) Elements() []element.Element {
+	return p.els
+}
+
+func (p *Parser) scan() bool {
+	if _, ok := p.tr.Read(1); !ok {
+		return true
+	}
+	p.tr.Unread(1)
+
+	type tryFunc func(element.ReadUnreader) (element.ParseResult, bool)
+	for _, tf := range []tryFunc{
+		element.NewCodeBlockFromTokens,
+		element.NewSubTitleFromTokens,
+		element.NewListFromTokens,
+		element.NewListItemFromTokens,
+		element.NewParagraphFromTokens,
+		element.NewStyleFromTokens,
+		element.NewLinkFromTokens,
+	} {
+		result, ok := tf(p.tr)
+		if !ok {
+			continue
+		}
+		el := result.Element
+		if len(result.Inner) != 0 {
+			par := NewParserFromChannel(token.NewTokenStream(result.Inner).Out())
+			par.Parse()
+			el = el.Append(par.Elements())
+		}
+		p.appendElement(el)
+		return false
+	}
+
+	p.doPlain()
+
+	return false
+}
+
+func (p *Parser) doPlain() {
+	tok, ok := p.tr.Read(1)
+	if !ok || tok[0].Type == token.TYPE_EOF || tok[0].Type == token.TYPE_EOS {
+		return
+	}
+	p.appendElement(element.MakePlain(tok[0]))
+}
+
+func (p *Parser) appendElement(b element.Element) {
+	p.els = append(p.els, b)
+}

parser/parser_test.go

@@ -1,10 +1,11 @@
-package adoc_test
+package parser_test
 
 import (
 	"strings"
 	"testing"
 
 	"ewintr.nl/adoc"
+	"ewintr.nl/adoc/parser"
 	"ewintr.nl/go-kit/test"
 )
 
@@ -20,7 +21,7 @@ func TestParser(t *testing.T) {
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			par := adoc.NewParser(strings.NewReader(tc.input))
+			par := parser.New(strings.NewReader(tc.input))
 			test.Equals(t, tc.exp, par.Parse())
 		})
 	}

styles.go

@@ -1,92 +0,0 @@
-package adoc
-
-type Strong []Element
-
-func (st Strong) Text() string {
-	var txt string
-	for _, e := range st {
-		txt += e.Text()
-	}
-
-	return txt
-}
-
-type Emphasis []Element
-
-func (em Emphasis) Text() string {
-	var txt string
-	for _, e := range em {
-		txt += e.Text()
-	}
-
-	return txt
-}
-
-type Code []Element
-
-func (c Code) Text() string {
-	var txt string
-	for _, e := range c {
-		txt += e.Text()
-	}
-
-	return txt
-}
-
-func (p *Parser) tryStyles() bool {
-	tok, ok := p.readN(2)
-	if !ok {
-		return false
-	}
-	markers := []Token{
-		{Type: TYPE_ASTERISK, Literal: "*"},
-		{Type: TYPE_UNDERSCORE, Literal: "_"},
-		{Type: TYPE_BACKTICK, Literal: "`"},
-	}
-	if !tok[0].Equal(markers...) {
-		p.unread(2)
-		return false
-	}
-	if tok[1].Type == TYPE_WHITESPACE {
-		p.unread(2)
-		return false
-	}
-
-	marker := tok[0]
-	toks := []Token{tok[1]}
-	for {
-		tok, ok := p.read()
-		if !ok {
-			p.unread(len(toks) + 1)
-			return false
-		}
-		if tok.Equal(TOKEN_EOF, TOKEN_DOUBLE_NEWLINE) {
-			p.unread(len(toks) + 1)
-			return false
-		}
-		if tok.Equal(marker) {
-			break
-		}
-		toks = append(toks, tok)
-	}
-	if toks[len(toks)-1].Type == TYPE_WHITESPACE {
-		p.unread(len(toks) + 2)
-		return false
-	}
-
-	par := NewParserFromChannel(NewTokenStream(toks).Out())
-	par.Parse()
-	var b Element
-	switch marker.Type {
-	case TYPE_ASTERISK:
-		b = Strong(par.Elements())
-	case TYPE_UNDERSCORE:
-		b = Emphasis(par.Elements())
-	case TYPE_BACKTICK:
-		b = Code(par.Elements())
-	}
-
-	p.appendElement(b)
-
-	return true
-}

styles_test.go

@@ -1,101 +0,0 @@
-package adoc_test
-
-import (
-	"strings"
-	"testing"
-
-	"ewintr.nl/adoc"
-	"ewintr.nl/go-kit/test"
-)
-
-func TestStyles(t *testing.T) {
-	for _, tc := range []struct {
-		name  string
-		input string
-		exp   []adoc.Element
-	}{
-		{
-			name:  "strong",
-			input: "*strong*",
-			exp: []adoc.Element{
-				adoc.Paragraph([]adoc.Element{
-					adoc.Strong{adoc.Word("strong")},
-				},
-				),
-			},
-		},
-		{
-			name:  "emphasis",
-			input: "_emphasis_",
-			exp: []adoc.Element{
-				adoc.Paragraph([]adoc.Element{
-					adoc.Emphasis{adoc.Word("emphasis")},
-				},
-				),
-			},
-		},
-		{
-			name:  "code",
-			input: "`code`",
-			exp: []adoc.Element{
-				adoc.Paragraph([]adoc.Element{
-					adoc.Code{adoc.Word("code")},
-				},
-				),
-			},
-		},
-		{
-			name:  "mixed",
-			input: "some `code code` in plain",
-			exp: []adoc.Element{
-				adoc.Paragraph([]adoc.Element{
-					adoc.Word("some"),
-					adoc.WhiteSpace(" "),
-					adoc.Code{
-						adoc.Word("code"),
-						adoc.WhiteSpace(" "),
-						adoc.Word("code"),
-					},
-					adoc.WhiteSpace(" "),
-					adoc.Word("in"),
-					adoc.WhiteSpace(" "),
-					adoc.Word("plain"),
-				},
-				),
-			},
-		},
-		{
-			name:  "incomplete",
-			input: "a *word",
-			exp: []adoc.Element{
-				adoc.Paragraph([]adoc.Element{
-					adoc.Word("a"),
-					adoc.WhiteSpace(" "),
-					adoc.Word("*"),
-					adoc.Word("word"),
-				}),
-			},
-		},
-		{
-			name:  "trailing space",
-			input: "*word *",
-			exp: []adoc.Element{
-				adoc.Paragraph([]adoc.Element{
-					adoc.Word("*"),
-					adoc.Word("word"),
-					adoc.WhiteSpace(" "),
-					adoc.Word("*"),
-				}),
-			},
-		},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			par := adoc.NewParser(strings.NewReader(tc.input))
-			exp := &adoc.ADoc{
-				Attributes: map[string]string{},
-				Content:    tc.exp,
-			}
-			test.Equals(t, exp, par.Parse())
-		})
-	}
-}

subtitle.go

@@ -1,52 +0,0 @@
-package adoc
-
-type SubTitle string
-
-func (st SubTitle) Text() string { return string(st) }
-
-type SubSubTitle string
-
-func (st SubSubTitle) Text() string { return string(st) }
-
-func (p *Parser) trySubTitle() bool {
-	toks, ok := p.readN(2)
-	if !ok {
-		return false
-	}
-	if toks[0].Type != TYPE_EQUALSIGN || toks[0].Len() < 2 || toks[1].Type != TYPE_WHITESPACE {
-		p.unread(2)
-		return false
-	}
-
-	for {
-		tok, ok := p.read()
-		if !ok {
-			p.unread(len(toks))
-			return false
-		}
-		if (tok.Type == TYPE_NEWLINE && tok.Len() > 1) || tok.Equal(TOKEN_EOF) {
-			break
-		}
-		toks = append(toks, tok)
-	}
-
-	var title string
-	for _, tok := range toks[2:] {
-		if tok.Type == TYPE_NEWLINE {
-			continue
-		}
-		title += p.makePlain(tok).Text()
-	}
-	switch toks[0].Len() {
-	case 2:
-		p.appendElement(SubTitle(title))
-	case 3:
-		p.appendElement(SubSubTitle(title))
-	default:
-		// ignore lower levels for now
-		p.unread(len(toks))
-		return false
-	}
-
-	return true
-}

token/lexer.go

@@ -1,4 +1,4 @@
-package adoc
+package token
 
 import (
 	"bufio"

token/lexer_test.go

@@ -1,32 +1,32 @@
-package adoc_test
+package token_test
 
 import (
 	"strings"
 	"testing"
 	"time"
 
-	"ewintr.nl/adoc"
+	"ewintr.nl/adoc/token"
 	"ewintr.nl/go-kit/test"
 )
 
 func TestLexer(t *testing.T) {
-	word := adoc.TYPE_WORD
+	word := token.TYPE_WORD
-	ws := adoc.TYPE_WHITESPACE
+	ws := token.TYPE_WHITESPACE
-	nl := adoc.TYPE_NEWLINE
+	nl := token.TYPE_NEWLINE
-	eq := adoc.TYPE_EQUALSIGN
+	eq := token.TYPE_EQUALSIGN
-	bt := adoc.TYPE_BACKTICK
+	bt := token.TYPE_BACKTICK
-	as := adoc.TYPE_ASTERISK
+	as := token.TYPE_ASTERISK
-	un := adoc.TYPE_UNDERSCORE
+	un := token.TYPE_UNDERSCORE
 
 	for _, tc := range []struct {
 		name  string
 		input string
-		exp   []adoc.Token
+		exp   []token.Token
 	}{
 		{
 			name:  "word string",
 			input: "one two",
-			exp: []adoc.Token{
+			exp: []token.Token{
 				{Type: word, Literal: "one"},
 				{Type: ws, Literal: " "},
 				{Type: word, Literal: "two"},
@@ -35,7 +35,7 @@ func TestLexer(t *testing.T) {
 		{
 			name:  "punctuation",
 			input: `. ,`,
-			exp: []adoc.Token{
+			exp: []token.Token{
 				{Type: word, Literal: "."},
 				{Type: ws, Literal: " "},
 				{Type: word, Literal: ","},
@@ -44,24 +44,24 @@ func TestLexer(t *testing.T) {
 		{
 			name:  "whitespace",
 			input: " \t",
-			exp: []adoc.Token{
+			exp: []token.Token{
 				{Type: ws, Literal: " \t"},
 			},
 		},
 		{
 			name:  "tab",
 			input: "\t",
-			exp:   []adoc.Token{{Type: ws, Literal: "\t"}},
+			exp:   []token.Token{{Type: ws, Literal: "\t"}},
 		},
 		{
 			name:  "newlines",
 			input: "\n\n\n",
-			exp:   []adoc.Token{{Type: nl, Literal: "\n\n\n"}},
+			exp:   []token.Token{{Type: nl, Literal: "\n\n\n"}},
 		},
 		{
 			name:  "special chars",
 			input: "=*_",
-			exp: []adoc.Token{
+			exp: []token.Token{
 				{Type: eq, Literal: "="},
 				{Type: as, Literal: "*"},
 				{Type: un, Literal: "_"},
@@ -70,7 +70,7 @@ func TestLexer(t *testing.T) {
 		{
 			name:  "mixed",
 			input: "This is a line with mixed \t `stuff`, see\t==?",
-			exp: []adoc.Token{
+			exp: []token.Token{
 				{Type: word, Literal: "This"},
 				{Type: ws, Literal: " "},
 				{Type: word, Literal: "is"},
@@ -97,8 +97,8 @@ func TestLexer(t *testing.T) {
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			input := strings.NewReader(tc.input)
-			lex := adoc.NewLexer(input)
+			lex := token.NewLexer(input)
-			act := []adoc.Token{}
+			act := []token.Token{}
 			stop := time.Now().Add(3 * time.Second)
 
 		T:
@@ -118,7 +118,7 @@ func TestLexer(t *testing.T) {
 			}
 
 			test.OK(t, lex.Error())
-			exp := append(tc.exp, adoc.TOKEN_EOF)
+			exp := append(tc.exp, token.TOKEN_EOF)
 			test.Equals(t, exp, act)
 		})
 	}

token/token.go

@@ -1,4 +1,4 @@
-package adoc
+package token
 
 const (
 	TYPE_EOF TokenType = iota

token/tokenreader.go

@@ -0,0 +1,65 @@
+package token
+
+type TokenReader struct {
+	in chan Token
+	ts []Token
+	r  int
+}
+
+func NewTokenReader(in chan Token) *TokenReader {
+	return &TokenReader{
+		in: in,
+		ts: []Token{},
+	}
+}
+
+func (tr *TokenReader) Read(n int) ([]Token, bool) {
+	toks := []Token{}
+	for i := 0; i < n; i++ {
+		tok, ok := tr.readOne()
+		if !ok {
+			tr.Unread(len(toks))
+			return []Token{}, false
+		}
+		toks = append(toks, tok)
+	}
+	return toks, true
+}
+
+func (tr *TokenReader) Unread(n int) bool {
+	//if n > tr.r {
+	//	return false
+	//}
+	for i := n; i > 0; i-- {
+		tr.r--
+	}
+	return true
+}
+
+func (tr *TokenReader) Discard() {
+	tr.ts = tr.ts[tr.r:]
+	tr.r = 0
+}
+
+func (tr *TokenReader) readOne() (Token, bool) {
+	if tr.r == len(tr.ts) {
+		if ok := tr.consume(); !ok {
+			return Token{}, false
+		}
+	}
+
+	tok := tr.ts[tr.r]
+	tr.r++
+
+	return tok, true
+}
+
+func (tr *TokenReader) consume() bool {
+	tok, ok := <-tr.in
+	if !ok {
+		return false
+	}
+	tr.ts = append(tr.ts, tok)
+
+	return true
+}

token/tokenstream.go

@@ -1,4 +1,4 @@
-package adoc
+package token
 
 type TokenStream struct {
 	ts  []Token
@@ -20,6 +20,7 @@ func (s *TokenStream) run() {
 	for _, tok := range s.ts {
 		s.out <- tok
 	}
+	s.out <- TOKEN_EOS
 	close(s.out)
 }