From d2b00c93e10a50416ee271316ea48896e32c69a0 Mon Sep 17 00:00:00 2001 From: Erik Winter Date: Tue, 15 Mar 2022 15:16:15 +0100 Subject: [PATCH] decouple packages --- adoc.go | 6 +- codeblock.go | 43 ------ element.go | 26 ---- element/codeblock.go | 54 +++++++ .../codeblock_test.go | 38 ++--- element/element.go | 31 ++++ element/header.go | 94 ++++++++++++ header_test.go => element/header_test.go | 20 +-- element/link.go | 72 +++++++++ link_test.go => element/link_test.go | 36 +++-- element/list.go | 56 +++++++ element/list_test.go | 79 ++++++++++ element/listitem.go | 54 +++++++ element/paragraph.go | 48 ++++++ element/paragraph_test.go | 78 ++++++++++ element/plain.go | 25 +++ element/styles.go | 106 +++++++++++++ element/styles_test.go | 103 +++++++++++++ element/subtitle.go | 62 ++++++++ subtitle_test.go => element/subtitle_test.go | 16 +- header.go | 76 --------- link.go | 65 -------- list.go | 77 ---------- list_test.go | 77 ---------- paragraph.go | 39 ----- paragraph_test.go | 76 --------- parser.go | 145 ------------------ parser/parser.go | 101 ++++++++++++ parser_test.go => parser/parser_test.go | 5 +- styles.go | 92 ----------- styles_test.go | 101 ------------ subtitle.go | 52 ------- lexer.go => token/lexer.go | 2 +- lexer_test.go => token/lexer_test.go | 40 ++--- token.go => token/token.go | 2 +- token/tokenreader.go | 65 ++++++++ tokenstream.go => token/tokenstream.go | 3 +- 37 files changed, 1118 insertions(+), 947 deletions(-) delete mode 100644 codeblock.go delete mode 100644 element.go create mode 100644 element/codeblock.go rename codeblock_test.go => element/codeblock_test.go (51%) create mode 100644 element/element.go create mode 100644 element/header.go rename header_test.go => element/header_test.go (72%) create mode 100644 element/link.go rename link_test.go => element/link_test.go (55%) create mode 100644 element/list.go create mode 100644 element/list_test.go create mode 100644 element/listitem.go create mode 100644 element/paragraph.go create mode 100644 element/paragraph_test.go create mode 100644 element/plain.go create mode 100644 element/styles.go create mode 100644 element/styles_test.go create mode 100644 element/subtitle.go rename subtitle_test.go => element/subtitle_test.go (60%) delete mode 100644 header.go delete mode 100644 link.go delete mode 100644 list.go delete mode 100644 list_test.go delete mode 100644 paragraph.go delete mode 100644 paragraph_test.go delete mode 100644 parser.go create mode 100644 parser/parser.go rename parser_test.go => parser/parser_test.go (78%) delete mode 100644 styles.go delete mode 100644 styles_test.go delete mode 100644 subtitle.go rename lexer.go => token/lexer.go (98%) rename lexer_test.go => token/lexer_test.go (77%) rename token.go => token/token.go (99%) create mode 100644 token/tokenreader.go rename tokenstream.go => token/tokenstream.go (91%) diff --git a/adoc.go b/adoc.go index b43634d..1c404b9 100644 --- a/adoc.go +++ b/adoc.go @@ -2,6 +2,8 @@ package adoc import ( "time" + + "ewintr.nl/adoc/element" ) type ADoc struct { @@ -10,12 +12,12 @@ type ADoc struct { Author string Path string Date time.Time - Content []Element + Content []element.Element } func NewADoc() *ADoc { return &ADoc{ Attributes: map[string]string{}, - Content: []Element{}, + Content: []element.Element{}, } } diff --git a/codeblock.go b/codeblock.go deleted file mode 100644 index 1c03ca9..0000000 --- a/codeblock.go +++ /dev/null @@ -1,43 +0,0 @@ -package adoc - -type CodeBlock []Element - -func (cb CodeBlock) Text() string { - txt := "" - for _, e := range cb { - txt += e.Text() - } - - return txt -} - -func (p *Parser) tryCodeBlock() bool { - delimiter := Token{Type: TYPE_DASH, Literal: "----"} - toks, ok := p.readN(2) - if !ok { - return false - } - if !toks[0].Equal(delimiter) || toks[1].Type != TYPE_NEWLINE { - p.unread(2) - return false - } - for { - tok, ok := p.read() - if !ok { - p.unread(len(toks)) - return false - } - if tok.Equal(delimiter) { - break - } - toks = append(toks, tok) - } - - cb := CodeBlock{} - for _, tok := range toks[2:] { - cb = append(cb, p.makePlain(tok)) - } - p.appendElement(cb) - - return true -} diff --git a/element.go b/element.go deleted file mode 100644 index 8d82420..0000000 --- a/element.go +++ /dev/null @@ -1,26 +0,0 @@ -package adoc - -type Element interface { - Text() string -} - -type Empty struct{} - -func (e Empty) Text() string { return "" } - -type Word string - -func (w Word) Text() string { return string(w) } - -type WhiteSpace string - -func (ws WhiteSpace) Text() string { return string(ws) } - -type Image struct { - Src string - Alt string -} - -func (i Image) Text() string { - return i.Alt -} diff --git a/element/codeblock.go b/element/codeblock.go new file mode 100644 index 0000000..5a0c01b --- /dev/null +++ b/element/codeblock.go @@ -0,0 +1,54 @@ +package element + +import ( + "ewintr.nl/adoc/token" +) + +type CodeBlock []Element + +func (cb CodeBlock) Text() string { + txt := "" + for _, e := range cb { + txt += e.Text() + } + + return txt +} + +func (cb CodeBlock) Append(els []Element) Element { + return CodeBlock{append(cb, els...)} +} + +func NewCodeBlockFromTokens(p ReadUnreader) (ParseResult, bool) { + delimiter := token.Token{Type: token.TYPE_DASH, Literal: "----"} + toks, ok := p.Read(2) + if !ok { + return ParseResult{}, false + } + if !toks[0].Equal(delimiter) || toks[1].Type != token.TYPE_NEWLINE { + p.Unread(2) + return ParseResult{}, false + } + for { + ntoks, ok := p.Read(1) + if !ok { + p.Unread(len(toks)) + return ParseResult{}, false + } + tok := ntoks[0] + if tok.Equal(delimiter) { + break + } + toks = append(toks, tok) + } + + cb := CodeBlock{} + for _, tok := range toks[2:] { + cb = append(cb, MakePlain(tok)) + } + + return ParseResult{ + Element: cb, + Inner: []token.Token{}, + }, true +} diff --git a/codeblock_test.go b/element/codeblock_test.go similarity index 51% rename from codeblock_test.go rename to element/codeblock_test.go index a30e1f7..38cb719 100644 --- a/codeblock_test.go +++ b/element/codeblock_test.go @@ -1,10 +1,12 @@ -package adoc_test +package element_test import ( "strings" "testing" "ewintr.nl/adoc" + "ewintr.nl/adoc/element" + "ewintr.nl/adoc/parser" "ewintr.nl/go-kit/test" ) @@ -20,7 +22,7 @@ func TestCodeBlock(t *testing.T) { ----`, exp: &adoc.ADoc{ Attributes: map[string]string{}, - Content: []adoc.Element{adoc.CodeBlock{}}, + Content: []element.Element{element.CodeBlock{}}, }, }, { @@ -32,11 +34,11 @@ more ----`, exp: &adoc.ADoc{ Attributes: map[string]string{}, - Content: []adoc.Element{adoc.CodeBlock{ - adoc.Word("code"), - adoc.WhiteSpace("\n\n"), - adoc.Word("more"), - adoc.WhiteSpace("\n"), + Content: []element.Element{element.CodeBlock{ + element.Word("code"), + element.WhiteSpace("\n\n"), + element.Word("more"), + element.WhiteSpace("\n"), }}, }, }, @@ -49,22 +51,22 @@ more `, exp: &adoc.ADoc{ Attributes: map[string]string{}, - Content: []adoc.Element{ - adoc.Paragraph{ - adoc.Word("----"), - adoc.WhiteSpace("\n"), - adoc.Word("code"), - }, - adoc.Paragraph{ - adoc.Word("more"), - adoc.WhiteSpace("\n"), - }, + Content: []element.Element{ + element.Paragraph{[]element.Element{ + element.Word("----"), + element.WhiteSpace("\n"), + element.Word("code"), + }}, + element.Paragraph{[]element.Element{ + element.Word("more"), + element.WhiteSpace("\n"), + }}, }, }, }, } { t.Run(tc.name, func(t *testing.T) { - par := adoc.NewParser(strings.NewReader(tc.input)) + par := parser.New(strings.NewReader(tc.input)) test.Equals(t, tc.exp, par.Parse()) }) } diff --git a/element/element.go b/element/element.go new file mode 100644 index 0000000..8fc4615 --- /dev/null +++ b/element/element.go @@ -0,0 +1,31 @@ +package element + +import "ewintr.nl/adoc/token" + +type Element interface { + Text() string + Append([]Element) Element +} + +type ParseResult struct { + Element Element + Inner []token.Token +} + +type ReadUnreader interface { + Read(n int) ([]token.Token, bool) + Unread(n int) bool +} + +type Empty struct{} + +func (e Empty) Text() string { return "" } + +type Image struct { + Src string + Alt string +} + +func (i Image) Text() string { + return i.Alt +} diff --git a/element/header.go b/element/header.go new file mode 100644 index 0000000..2cc466b --- /dev/null +++ b/element/header.go @@ -0,0 +1,94 @@ +package element + +import ( + "time" + + "ewintr.nl/adoc/token" +) + +type Header struct { + Title string + Author string + Date time.Time + Attributes map[string]string +} + +func (h Header) Text() string { return h.Title } +func (h Header) Append(_ []Element) Element { return h } + +func NewHeaderFromTokens(tr ReadUnreader) (ParseResult, bool) { + toks, ok := tr.Read(2) + if !ok { + return ParseResult{}, false + } + if !toks[0].Equal(token.Token{Type: token.TYPE_EQUALSIGN, Literal: "="}) { + tr.Unread(2) + return ParseResult{}, false + } + if toks[1].Type != token.TYPE_WHITESPACE { + tr.Unread(2) + return ParseResult{}, false + } + for { + ntoks, ok := tr.Read(1) + if !ok { + tr.Unread(len(toks)) + return ParseResult{}, false + } + if ntoks[0].Equal(token.TOKEN_EOF, token.TOKEN_DOUBLE_NEWLINE) { + break + } + toks = append(toks, ntoks[0]) + } + + h := &Header{ + Attributes: map[string]string{}, + } + lines := token.Split(toks[2:], token.TYPE_NEWLINE) + h.Title = token.Literals(lines[0]) + + for _, line := range lines[1:] { + switch { + case tryHeaderDate(h, line): + continue + case tryHeaderField(h, line): + continue + default: + h.Author = token.Literals(line) + } + } + + return ParseResult{ + Element: *h, + Inner: []token.Token{}, + }, true +} + +func tryHeaderField(h *Header, line []token.Token) bool { + if len(line) < 4 { + return false + } + pair := token.Split(line, token.TYPE_WHITESPACE) + if len(pair) != 2 { + return false + } + key, value := pair[0], pair[1] + + if !token.HasPattern(key, []token.TokenType{token.TYPE_COLON, token.TYPE_WORD, token.TYPE_COLON}) { + return false + } + + h.Attributes[key[1].Literal] = token.Literals(value) + + return true +} + +func tryHeaderDate(h *Header, line []token.Token) bool { + date, err := time.Parse("2006-01-02", token.Literals(line)) + if err != nil { + return false + } + h.Date = date + + return true +} diff --git a/header_test.go b/element/header_test.go similarity index 72% rename from header_test.go rename to element/header_test.go index ccf0432..54a6c2a 100644 --- a/header_test.go +++ b/element/header_test.go @@ -1,4 +1,4 @@ -package adoc_test +package element_test import ( "strings" @@ -6,6 +6,8 @@ import ( "time" "ewintr.nl/adoc" + "ewintr.nl/adoc/element" + "ewintr.nl/adoc/parser" "ewintr.nl/go-kit/test" ) @@ -21,7 +23,7 @@ func TestHeader(t *testing.T) { exp: &adoc.ADoc{ Title: "Title", Attributes: map[string]string{}, - Content: []adoc.Element{}, + Content: []element.Element{}, }, }, { @@ -46,18 +48,18 @@ First paragraph`, "key1": "value1", "key2": "value2", }, - Content: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Word("First"), - adoc.WhiteSpace(" "), - adoc.Word("paragraph"), - }), + Content: []element.Element{ + element.Paragraph{[]element.Element{ + element.Word("First"), + element.WhiteSpace(" "), + element.Word("paragraph"), + }}, }, }, }, } { t.Run(tc.name, func(t *testing.T) { - par := adoc.NewParser(strings.NewReader(tc.input)) + par := parser.New(strings.NewReader(tc.input)) test.Equals(t, tc.exp, par.Parse()) }) } diff --git a/element/link.go b/element/link.go new file mode 100644 index 0000000..b2d907f --- /dev/null +++ b/element/link.go @@ -0,0 +1,72 @@ +package element + +import ( + "ewintr.nl/adoc/token" +) + +type Link struct { + URL string + Title string +} + +func (l Link) Text() string { return l.Title } +func (l Link) Append(_ []Element) Element { return l } + +func NewLinkFromTokens(tr ReadUnreader) (ParseResult, bool) { + tok, ok := tr.Read(1) + if !ok { + return ParseResult{}, false + } + tr.Unread(1) + if tok[0].Type != token.TYPE_WORD { + return ParseResult{}, false + } + + url, title := []token.Token{}, []token.Token{} + lb := false + count := 0 + for { + ntoks, ok := tr.Read(1) + if !ok { + tr.Unread(count) + return ParseResult{}, false + } + count++ + tok := ntoks[0] + if tok.Equal(token.TOKEN_EOF, token.TOKEN_EOS, token.TOKEN_NEWLINE, token.TOKEN_DOUBLE_NEWLINE) { + tr.Unread(count) + return ParseResult{}, false + } + if !lb && tok.Type == token.TYPE_WHITESPACE { + tr.Unread(count) + return ParseResult{}, false + } + if tok.Equal(token.Token{Type: token.TYPE_BRACKETOPEN, Literal: "["}) { + lb = true + continue + } + if lb && tok.Equal(token.Token{Type: token.TYPE_BRACKETCLOSE, Literal: "]"}) { + break + } + + if lb { + title = append(title, tok) + continue + } + url = append(url, tok) + } + + if len(url) == 0 || !lb { + tr.Unread(count) + return ParseResult{}, false + } + link := Link{ + URL: token.Literals(url), + Title: token.Literals(title), + } + + return ParseResult{ + Element: link, + Inner: []token.Token{}, + }, true +} diff --git a/link_test.go b/element/link_test.go similarity index 55% rename from link_test.go rename to element/link_test.go index 564f195..ff2ffc8 100644 --- a/link_test.go +++ b/element/link_test.go @@ -1,10 +1,12 @@ -package adoc_test +package element_test import ( "strings" "testing" "ewintr.nl/adoc" + "ewintr.nl/adoc/element" + "ewintr.nl/adoc/parser" "ewintr.nl/go-kit/test" ) @@ -12,41 +14,41 @@ func TestLink(t *testing.T) { for _, tc := range []struct { name string input string - exp []adoc.Element + exp []element.Element }{ { name: "simple", input: "a link[title] somewhere", - exp: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Word("a"), - adoc.WhiteSpace(" "), - adoc.Link{ + exp: []element.Element{ + element.Paragraph{Elements: []element.Element{ + element.Word("a"), + element.WhiteSpace(" "), + element.Link{ URL: "link", Title: "title", }, - adoc.WhiteSpace(" "), - adoc.Word("somewhere"), - }), + element.WhiteSpace(" "), + element.Word("somewhere"), + }}, }, }, { name: "with underscore", input: "check https://example.com/some_url[some url]", - exp: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Word("check"), - adoc.WhiteSpace(" "), - adoc.Link{ + exp: []element.Element{ + element.Paragraph{Elements: []element.Element{ + element.Word("check"), + element.WhiteSpace(" "), + element.Link{ URL: "https://example.com/some_url", Title: "some url", }, - }), + }}, }, }, } { t.Run(tc.name, func(t *testing.T) { - par := adoc.NewParser(strings.NewReader(tc.input)) + par := parser.New(strings.NewReader(tc.input)) exp := &adoc.ADoc{ Attributes: map[string]string{}, Content: tc.exp, diff --git a/element/list.go b/element/list.go new file mode 100644 index 0000000..bfc6f3f --- /dev/null +++ b/element/list.go @@ -0,0 +1,56 @@ +package element + +import ( + "ewintr.nl/adoc/token" +) + +type List []ListItem + +func (l List) Text() string { + txt := "" + for _, li := range l { + txt += li.Text() + } + + return txt +} + +func (l List) Append(els []Element) Element { + for _, el := range els { + if val, ok := el.(ListItem); ok { + l = append(l, val) + } + } + return l +} + +func NewListFromTokens(tr ReadUnreader) (ParseResult, bool) { + toks, ok := tr.Read(2) + if !ok { + return ParseResult{}, false + } + if !toks[0].Equal(token.TOKEN_ASTERISK) || toks[1].Type != token.TYPE_WHITESPACE { + tr.Unread(2) + return ParseResult{}, false + } + + tr.Unread(2) + toks = []token.Token{} + for { + ntoks, ok := tr.Read(1) + if !ok { + tr.Unread(len(toks)) + return ParseResult{}, false + } + tok := ntoks[0] + if tok.Equal(token.TOKEN_DOUBLE_NEWLINE, token.TOKEN_EOF) { + break + } + toks = append(toks, tok) + } + + return ParseResult{ + Element: List{}, + Inner: toks, + }, true +} diff --git a/element/list_test.go b/element/list_test.go new file mode 100644 index 0000000..ee18aca --- /dev/null +++ b/element/list_test.go @@ -0,0 +1,79 @@ +package element_test + +import ( + "strings" + "testing" + + "ewintr.nl/adoc" + "ewintr.nl/adoc/element" + "ewintr.nl/adoc/parser" + "ewintr.nl/go-kit/test" +) + +func TestList(t *testing.T) { + for _, tc := range []struct { + name string + input string + exp []element.Element + }{ + { + name: "one item", + input: `* item 1`, + exp: []element.Element{ + element.List([]element.ListItem{ + {element.Word("item"), element.WhiteSpace(" "), element.Word("1")}, + }, + )}, + }, + { + name: "multiple", + input: `* item 1 +* item 2 +* item 3`, + exp: []element.Element{ + element.List([]element.ListItem{ + {element.Word("item"), element.WhiteSpace(" "), element.Word("1")}, + {element.Word("item"), element.WhiteSpace(" "), element.Word("2")}, + {element.Word("item"), element.WhiteSpace(" "), element.Word("3")}, + })}, + }, + { + name: "double with pararaph", + input: `* item 1 + +* item 2 +* item 3 + +and some text`, + exp: []element.Element{ + element.List( + []element.ListItem{ + {element.Word("item"), element.WhiteSpace(" "), element.Word("1")}, + }, + ), + element.List( + []element.ListItem{ + {element.Word("item"), element.WhiteSpace(" "), element.Word("2")}, + {element.Word("item"), element.WhiteSpace(" "), element.Word("3")}, + }, + ), + element.Paragraph{Elements: []element.Element{ + element.Word("and"), + element.WhiteSpace(" "), + element.Word("some"), + element.WhiteSpace(" "), + element.Word("text"), + }}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + par := parser.New(strings.NewReader(tc.input)) + exp := &adoc.ADoc{ + Attributes: map[string]string{}, + Content: tc.exp, + } + test.Equals(t, exp, par.Parse()) + }) + } +} diff --git a/element/listitem.go b/element/listitem.go new file mode 100644 index 0000000..caf0dbe --- /dev/null +++ b/element/listitem.go @@ -0,0 +1,54 @@ +package element + +import ( + "ewintr.nl/adoc/token" +) + +type ListItem []Element + +func (li ListItem) Text() string { + txt := "" + for _, e := range li { + txt += e.Text() + } + + return txt +} + +func (li ListItem) Append(els []Element) Element { + for _, el := range els { + li = append(li, el) + } + + return li +} + +func NewListItemFromTokens(tr ReadUnreader) (ParseResult, bool) { + toks, ok := tr.Read(2) + if !ok { + return ParseResult{}, false + } + if !toks[0].Equal(token.TOKEN_ASTERISK) || toks[1].Type != token.TYPE_WHITESPACE { + tr.Unread(2) + return ParseResult{}, false + } + + toks = []token.Token{} + for { + ntoks, ok := tr.Read(1) + if !ok { + tr.Unread(len(toks)) + return ParseResult{}, false + } + tok := ntoks[0] + if tok.Equal(token.TOKEN_NEWLINE, token.TOKEN_EOS) { + break + } + toks = append(toks, tok) + } + + return ParseResult{ + Element: ListItem{}, + Inner: toks, + }, true +} diff --git a/element/paragraph.go b/element/paragraph.go new file mode 100644 index 0000000..82fd61c --- /dev/null +++ b/element/paragraph.go @@ -0,0 +1,48 @@ +package element + +import ( + "ewintr.nl/adoc/token" +) + +type Paragraph struct { + Elements []Element +} + +func (p Paragraph) Text() string { + txt := "" + for _, el := range p.Elements { + txt += el.Text() + } + + return txt +} + +func (p Paragraph) Append(els []Element) Element { + return Paragraph{ + Elements: append(p.Elements, els...), + } +} + +func NewParagraphFromTokens(tr ReadUnreader) (ParseResult, bool) { + toks := []token.Token{} + for { + tok, ok := tr.Read(1) + if !ok { + tr.Unread(len(toks)) + return ParseResult{}, false + } + if tok[0].Equal(token.TOKEN_EOF, token.TOKEN_DOUBLE_NEWLINE) { + break + } + toks = append(toks, tok[0]) + } + + if len(toks) == 0 { + return ParseResult{}, false + } + + return ParseResult{ + Element: Paragraph{Elements: []Element{}}, + Inner: toks, + }, true +} diff --git a/element/paragraph_test.go b/element/paragraph_test.go new file mode 100644 index 0000000..101d7a6 --- /dev/null +++ b/element/paragraph_test.go @@ -0,0 +1,78 @@ +package element_test + +import ( + "strings" + "testing" + + "ewintr.nl/adoc" + "ewintr.nl/adoc/element" + "ewintr.nl/adoc/parser" + "ewintr.nl/go-kit/test" +) + +func TestParagraph(t *testing.T) { + for _, tc := range []struct { + name string + input string + exp *adoc.ADoc + }{ + { + name: "single paragraph", + input: "some text", + exp: &adoc.ADoc{ + Attributes: map[string]string{}, + Content: []element.Element{ + element.Paragraph{Elements: []element.Element{ + element.Word("some"), + element.WhiteSpace(" "), + element.Word("text"), + }}, + }}, + }, + { + name: "title with paragraphs", + input: `= Title + +paragraph one + +paragraph two`, + exp: &adoc.ADoc{ + Title: "Title", + Attributes: map[string]string{}, + Content: []element.Element{ + element.Paragraph{Elements: []element.Element{ + element.Word("paragraph"), + element.WhiteSpace(" "), + element.Word("one"), + }}, + element.Paragraph{Elements: []element.Element{ + element.Word("paragraph"), + element.WhiteSpace(" "), + element.Word("two"), + }}, + }, + }, + }, + { + name: "three with trailing newline", + input: `one + +two + +three +`, + exp: &adoc.ADoc{ + Attributes: map[string]string{}, + Content: []element.Element{ + element.Paragraph{Elements: []element.Element{element.Word("one")}}, + element.Paragraph{Elements: []element.Element{element.Word("two")}}, + element.Paragraph{Elements: []element.Element{element.Word("three"), element.WhiteSpace("\n")}}, + }}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + par := parser.New(strings.NewReader(tc.input)) + test.Equals(t, tc.exp, par.Parse()) + }) + } +} diff --git a/element/plain.go b/element/plain.go new file mode 100644 index 0000000..6d0935f --- /dev/null +++ b/element/plain.go @@ -0,0 +1,25 @@ +package element + +import "ewintr.nl/adoc/token" + +type Word string + +func (w Word) Text() string { return string(w) } +func (w Word) Append(_ []Element) Element { return w } + +type WhiteSpace string + +func (ws WhiteSpace) Text() string { return string(ws) } +func (ws WhiteSpace) Append(_ []Element) Element { return ws } + +func MakePlain(tok token.Token) Element { + switch tok.Type { + case token.TYPE_WHITESPACE: + return WhiteSpace(tok.Literal) + case token.TYPE_NEWLINE: + return WhiteSpace(tok.Literal) + default: + return Word(tok.Literal) + } + +} diff --git a/element/styles.go b/element/styles.go new file mode 100644 index 0000000..1f23eb9 --- /dev/null +++ b/element/styles.go @@ -0,0 +1,106 @@ +package element + +import "ewintr.nl/adoc/token" + +type Strong []Element + +func (st Strong) Text() string { + var txt string + for _, e := range st { + txt += e.Text() + } + + return txt +} + +func (st Strong) Append(els []Element) Element { + return append(st, els...) +} + +type Emphasis []Element + +func (em Emphasis) Text() string { + var txt string + for _, e := range em { + txt += e.Text() + } + + return txt +} + +func (em Emphasis) Append(els []Element) Element { + return append(em, els...) +} + +type Code []Element + +func (c Code) Text() string { + var txt string + for _, e := range c { + txt += e.Text() + } + + return txt +} + +func (c Code) Append(els []Element) Element { + return append(c, els...) +} + +func NewStyleFromTokens(tr ReadUnreader) (ParseResult, bool) { + toks, ok := tr.Read(2) + if !ok { + return ParseResult{}, false + } + markers := []token.Token{ + {Type: token.TYPE_ASTERISK, Literal: "*"}, + {Type: token.TYPE_UNDERSCORE, Literal: "_"}, + {Type: token.TYPE_BACKTICK, Literal: "`"}, + } + if !toks[0].Equal(markers...) { + tr.Unread(2) + return ParseResult{}, false + } + if toks[1].Type == token.TYPE_WHITESPACE { + tr.Unread(2) + return ParseResult{}, false + } + + marker := toks[0] + toks = []token.Token{toks[1]} + for { + ntoks, ok := tr.Read(1) + if !ok { + tr.Unread(len(toks) + 1) + return ParseResult{}, false + } + tok := ntoks[0] + if tok.Equal(token.TOKEN_EOF, token.TOKEN_DOUBLE_NEWLINE) { + tr.Unread(len(toks) + 1) + return ParseResult{}, false + } + if tok.Equal(marker) { + break + } + toks = append(toks, tok) + } + if toks[len(toks)-1].Type == token.TYPE_WHITESPACE { + tr.Unread(len(toks) + 2) + return ParseResult{}, false + } + + var s Element + switch marker.Type { + case token.TYPE_ASTERISK: + s = Strong{} + case token.TYPE_UNDERSCORE: + s = Emphasis{} + case token.TYPE_BACKTICK: + s = Code{} + } + + return ParseResult{ + Element: s, + Inner: toks, + }, true +} diff --git a/element/styles_test.go b/element/styles_test.go new file mode 100644 index 0000000..53ac533 --- /dev/null +++ b/element/styles_test.go @@ -0,0 +1,103 @@ +package element_test + +import ( + "strings" + "testing" + + "ewintr.nl/adoc" + "ewintr.nl/adoc/element" + "ewintr.nl/adoc/parser" + "ewintr.nl/go-kit/test" +) + +func TestStyles(t *testing.T) { + for _, tc := range []struct { + name string + input string + exp []element.Element + }{ + { + name: "strong", + input: "*strong*", + exp: []element.Element{ + element.Paragraph{Elements: []element.Element{ + element.Strong{element.Word("strong")}, + }, + }, + }, + }, + { + name: "emphasis", + input: "_emphasis_", + exp: []element.Element{ + element.Paragraph{Elements: []element.Element{ + element.Emphasis{element.Word("emphasis")}, + }, + }, + }, + }, + { + name: "code", + input: "`code`", + exp: []element.Element{ + element.Paragraph{Elements: []element.Element{ + element.Code{element.Word("code")}, + }, + }, + }, + }, + { + name: "mixed", + input: "some `code code` in plain", + exp: []element.Element{ + element.Paragraph{Elements: []element.Element{ + element.Word("some"), + element.WhiteSpace(" "), + element.Code{ + element.Word("code"), + element.WhiteSpace(" "), + element.Word("code"), + }, + element.WhiteSpace(" "), + element.Word("in"), + element.WhiteSpace(" "), + element.Word("plain"), + }, + }, + }, + }, + { + name: "incomplete", + input: "a *word", + exp: []element.Element{ + element.Paragraph{Elements: []element.Element{ + element.Word("a"), + element.WhiteSpace(" "), + element.Word("*"), + element.Word("word"), + }}, + }, + }, + { + name: "trailing space", + input: "*word *", + exp: []element.Element{ + element.Paragraph{Elements: []element.Element{ + element.Word("*"), + element.Word("word"), + element.WhiteSpace(" "), + element.Word("*"), + }}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + par := parser.New(strings.NewReader(tc.input)) + exp := &adoc.ADoc{ + Attributes: map[string]string{}, + Content: tc.exp, + } + test.Equals(t, exp, par.Parse()) + }) + } +} diff --git a/element/subtitle.go b/element/subtitle.go new file mode 100644 index 0000000..ad0f8d6 --- /dev/null +++ b/element/subtitle.go @@ -0,0 +1,62 @@ +package element + +import "ewintr.nl/adoc/token" + +type SubTitle string + +func (st SubTitle) Text() string { return string(st) } +func (st SubTitle) Append(_ []Element) Element { return st } + +type SubSubTitle string + +func (st SubSubTitle) Text() string { return string(st) } +func (st SubSubTitle) Append(_ []Element) Element { return st } + +func NewSubTitleFromTokens(tr ReadUnreader) (ParseResult, bool) { + toks, ok := tr.Read(2) + if !ok { + return ParseResult{}, false + } + if toks[0].Type != token.TYPE_EQUALSIGN || toks[0].Len() < 2 || toks[1].Type != token.TYPE_WHITESPACE { + tr.Unread(2) + return ParseResult{}, false + } + + for { + ntoks, ok := tr.Read(1) + if !ok { + tr.Unread(len(toks)) + return ParseResult{}, false + } + tok := ntoks[0] + if (tok.Type == token.TYPE_NEWLINE && tok.Len() > 1) || tok.Equal(token.TOKEN_EOF) { + break + } + toks = append(toks, tok) + } + + var title string + for _, tok := range toks[2:] { + if tok.Type == token.TYPE_NEWLINE { + continue + } + title += MakePlain(tok).Text() + } + + var el Element + switch toks[0].Len() { + case 2: + el = SubTitle(title) + case 3: + el = SubSubTitle(title) + default: + // ignore lower levels for now + tr.Unread(len(toks)) + return ParseResult{}, false + } + + return ParseResult{ + Element: el, + Inner: []token.Token{}, + }, true +} diff --git a/subtitle_test.go b/element/subtitle_test.go similarity index 60% rename from subtitle_test.go rename to element/subtitle_test.go index ea57cb8..e0a0f7d 100644 --- a/subtitle_test.go +++ b/element/subtitle_test.go @@ -1,10 +1,12 @@ -package adoc_test +package element_test import ( "strings" "testing" "ewintr.nl/adoc" + "ewintr.nl/adoc/element" + "ewintr.nl/adoc/parser" "ewintr.nl/go-kit/test" ) @@ -12,27 +14,27 @@ func TestSubTitle(t *testing.T) { for _, tc := range []struct { name string input string - exp []adoc.Element + exp []element.Element }{ { name: "empty", input: "== ", - exp: []adoc.Element{adoc.SubTitle("")}, + exp: []element.Element{element.SubTitle("")}, }, { name: "subtitle", input: "== title with words", - exp: []adoc.Element{adoc.SubTitle("title with words")}, + exp: []element.Element{element.SubTitle("title with words")}, }, { name: "subsubtitle", input: "=== title", - exp: []adoc.Element{adoc.SubSubTitle("title")}, + exp: []element.Element{element.SubSubTitle("title")}, }, { name: "trailing newline", input: "== title\n", - exp: []adoc.Element{adoc.SubTitle("title")}, + exp: []element.Element{element.SubTitle("title")}, }, } { t.Run(tc.name, func(t *testing.T) { @@ -40,7 +42,7 @@ func TestSubTitle(t *testing.T) { Attributes: map[string]string{}, Content: tc.exp, } - par := adoc.NewParser(strings.NewReader(tc.input)) + par := parser.New(strings.NewReader(tc.input)) test.Equals(t, exp, par.Parse()) }) } diff --git a/header.go b/header.go deleted file mode 100644 index 80d03c1..0000000 --- a/header.go +++ /dev/null @@ -1,76 +0,0 @@ -package adoc - -import ( - "time" -) - -func (p *Parser) tryHeader() bool { - toks, ok := p.readN(2) - if !ok { - return false - } - if !toks[0].Equal(Token{Type: TYPE_EQUALSIGN, Literal: "="}) { - p.unread(2) - return false - } - if toks[1].Type != TYPE_WHITESPACE { - p.unread(2) - return false - } - for { - tok, ok := p.read() - if !ok { - p.unread(len(toks)) - return false - } - if tok.Equal(TOKEN_EOF, TOKEN_DOUBLE_NEWLINE) { - break - } - toks = append(toks, tok) - } - - lines := Split(toks[2:], TYPE_NEWLINE) - p.doc.Title = Literals(lines[0]) - - for _, line := range lines[1:] { - switch { - case p.tryHeaderDate(line): - continue - case p.tryHeaderField(line): - continue - default: - p.doc.Author = Literals(line) - } - } - - return true -} - -func (p *Parser) tryHeaderField(line []Token) bool { - if len(line) < 4 { - return false - } - pair := Split(line, TYPE_WHITESPACE) - if len(pair) != 2 { - return false - } - key, value := pair[0], pair[1] - - if !HasPattern(key, []TokenType{TYPE_COLON, TYPE_WORD, TYPE_COLON}) { - return false - } - - p.doc.Attributes[key[1].Literal] = Literals(value) - - return true -} - -func (p *Parser) tryHeaderDate(line []Token) bool { - date, err := time.Parse("2006-01-02", Literals(line)) - if err != nil { - return false - } - p.doc.Date = date - - return true -} diff --git a/link.go b/link.go deleted file mode 100644 index 8f97313..0000000 --- a/link.go +++ /dev/null @@ -1,65 +0,0 @@ -package adoc - -import "fmt" - -type Link struct { - URL string - Title string -} - -func (l Link) Text() string { return l.Title } - -func (p *Parser) tryLink() bool { - tok, ok := p.peek() - if !ok { - return false - } - if tok.Type != TYPE_WORD { - return false - } - - url, title := []Token{}, []Token{} - lb := false - count := 0 - for { - tok, ok := p.read() - if !ok { - p.unread(count) - return false - } - count++ - if tok.Equal(TOKEN_EOF, TOKEN_NEWLINE, TOKEN_DOUBLE_NEWLINE) { - p.unread(count) - return false - } - if !lb && tok.Type == TYPE_WHITESPACE { - p.unread(count) - return false - } - if tok.Equal(Token{Type: TYPE_BRACKETOPEN, Literal: "["}) { - lb = true - continue - } - if tok.Equal(Token{Type: TYPE_BRACKETCLOSE, Literal: "]"}) { - break - } - - if lb { - title = append(title, tok) - continue - } - url = append(url, tok) - } - fmt.Printf("url: %+v, title: %+v\n", url, title) - - if len(url) == 0 || !lb { - p.unread(count) - return false - } - link := Link{ - URL: Literals(url), - Title: Literals(title), - } - p.appendElement(link) - return true -} diff --git a/list.go b/list.go deleted file mode 100644 index cca89d3..0000000 --- a/list.go +++ /dev/null @@ -1,77 +0,0 @@ -package adoc - -type ListItem []Element - -func (li ListItem) Text() string { - txt := "" - for _, e := range li { - txt += e.Text() - } - - return txt -} - -type List []ListItem - -func (l List) Text() string { - txt := "" - for _, li := range l { - txt += li.Text() - } - - return txt -} - -func (p *Parser) tryList() bool { - toks, ok := p.readN(2) - if !ok { - return false - } - if !toks[0].Equal(TOKEN_ASTERISK) || toks[1].Type != TYPE_WHITESPACE { - p.unread(2) - return false - } - - p.unread(2) - toks = []Token{} - toksCount := 0 - var items []ListItem - for { - tok, ok := p.read() - if !ok { - p.unread(toksCount) - return false - } - if tok.Equal(TOKEN_NEWLINE, TOKEN_DOUBLE_NEWLINE, TOKEN_EOF) { - item, ok := tryListItem(toks) - if !ok { - p.unread(len(toks)) - return false - } - items = append(items, item) - toks = []Token{} - if tok.Equal(TOKEN_DOUBLE_NEWLINE, TOKEN_EOF) { - break - } - continue - } - toks = append(toks, tok) - toksCount++ - } - - p.appendElement(List(items)) - - return true -} - -func tryListItem(toks []Token) (ListItem, bool) { - if !toks[0].Equal(TOKEN_ASTERISK) || toks[1].Type != TYPE_WHITESPACE { - return ListItem{}, false - } - stream := NewTokenStream(toks[2:]).Out() - par := NewParserFromChannel(stream) - par.Parse() - b := ListItem(par.Elements()) - - return b, true -} diff --git a/list_test.go b/list_test.go deleted file mode 100644 index 79e16f0..0000000 --- a/list_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package adoc_test - -import ( - "strings" - "testing" - - "ewintr.nl/adoc" - "ewintr.nl/go-kit/test" -) - -func TestList(t *testing.T) { - for _, tc := range []struct { - name string - input string - exp []adoc.Element - }{ - { - name: "one item", - input: `* item 1`, - exp: []adoc.Element{ - adoc.List([]adoc.ListItem{ - {adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("1")}, - }, - )}, - }, - { - name: "multiple", - input: `* item 1 -* item 2 -* item 3`, - exp: []adoc.Element{ - adoc.List([]adoc.ListItem{ - {adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("1")}, - {adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("2")}, - {adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("3")}, - })}, - }, - { - name: "double with pararaph", - input: `* item 1 - -* item 2 -* item 3 - -and some text`, - exp: []adoc.Element{ - adoc.List( - []adoc.ListItem{ - {adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("1")}, - }, - ), - adoc.List( - []adoc.ListItem{ - {adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("2")}, - {adoc.Word("item"), adoc.WhiteSpace(" "), adoc.Word("3")}, - }, - ), - adoc.Paragraph([]adoc.Element{ - adoc.Word("and"), - adoc.WhiteSpace(" "), - adoc.Word("some"), - adoc.WhiteSpace(" "), - adoc.Word("text"), - }), - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - par := adoc.NewParser(strings.NewReader(tc.input)) - exp := &adoc.ADoc{ - Attributes: map[string]string{}, - Content: tc.exp, - } - test.Equals(t, exp, par.Parse()) - }) - } -} diff --git a/paragraph.go b/paragraph.go deleted file mode 100644 index 597a173..0000000 --- a/paragraph.go +++ /dev/null @@ -1,39 +0,0 @@ -package adoc - -type Paragraph []Element - -func (p Paragraph) Text() string { - txt := "" - for _, e := range p { - txt += e.Text() - } - - return txt -} - -func (p *Parser) tryParagraph() bool { - toks := []Token{} - for { - tok, ok := p.read() - if !ok { - p.unread(len(toks)) - return false - } - if tok.Equal(TOKEN_EOF, TOKEN_DOUBLE_NEWLINE) { - break - } - toks = append(toks, tok) - } - - if len(toks) == 0 { - return false - } - - par := NewParserFromChannel(NewTokenStream(toks).Out()) - par.Parse() - b := Paragraph(par.Elements()) - - p.appendElement(b) - - return true -} diff --git a/paragraph_test.go b/paragraph_test.go deleted file mode 100644 index 73f31be..0000000 --- a/paragraph_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package adoc_test - -import ( - "strings" - "testing" - - "ewintr.nl/adoc" - "ewintr.nl/go-kit/test" -) - -func TestParagraph(t *testing.T) { - for _, tc := range []struct { - name string - input string - exp *adoc.ADoc - }{ - { - name: "single paragraph", - input: "some text", - exp: &adoc.ADoc{ - Attributes: map[string]string{}, - Content: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Word("some"), - adoc.WhiteSpace(" "), - adoc.Word("text"), - }), - }}, - }, - { - name: "title with paragraphs", - input: `= Title - -paragraph one - -paragraph two`, - exp: &adoc.ADoc{ - Title: "Title", - Attributes: map[string]string{}, - Content: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Word("paragraph"), - adoc.WhiteSpace(" "), - adoc.Word("one"), - }), - adoc.Paragraph([]adoc.Element{ - adoc.Word("paragraph"), - adoc.WhiteSpace(" "), - adoc.Word("two"), - }), - }, - }, - }, - { - name: "three with trailing newline", - input: `one - -two - -three -`, - exp: &adoc.ADoc{ - Attributes: map[string]string{}, - Content: []adoc.Element{ - adoc.Paragraph([]adoc.Element{adoc.Word("one")}), - adoc.Paragraph([]adoc.Element{adoc.Word("two")}), - adoc.Paragraph([]adoc.Element{adoc.Word("three"), adoc.WhiteSpace("\n")}), - }}, - }, - } { - t.Run(tc.name, func(t *testing.T) { - par := adoc.NewParser(strings.NewReader(tc.input)) - test.Equals(t, tc.exp, par.Parse()) - }) - } -} diff --git a/parser.go b/parser.go deleted file mode 100644 index 0e5a087..0000000 --- a/parser.go +++ /dev/null @@ -1,145 +0,0 @@ -package adoc - -import "io" - -type Parser struct { - in chan Token - doc *ADoc - ts []Token - r int - els []Element -} - -func NewParser(reader io.Reader) *Parser { - lex := NewLexer(reader) - return NewParserFromChannel(lex.Out()) -} - -func NewParserFromChannel(toks chan Token) *Parser { - return &Parser{ - in: toks, - doc: NewADoc(), - ts: []Token{}, - els: []Element{}, - } -} - -func (p *Parser) Parse() *ADoc { - p.tryHeader() - - for { - if done := p.scan(); done { - break - } - p.discard() - } - - p.doc.Content = p.els - return p.doc -} - -func (p *Parser) Elements() []Element { - return p.els -} - -func (p *Parser) scan() bool { - if _, ok := p.peek(); !ok { - return true - } - - type tryFunc func() bool - for _, tf := range []tryFunc{ - p.tryCodeBlock, p.trySubTitle, p.tryList, p.tryParagraph, - p.tryStyles, p.tryLink, - } { - if tf() { - return false - } - } - - p.doPlain() - - return false -} - -func (p *Parser) doPlain() { - tok, ok := p.read() - if !ok { - return - } - p.appendElement(p.makePlain(tok)) -} - -func (p *Parser) makePlain(tok Token) Element { - switch tok.Type { - case TYPE_WHITESPACE: - return WhiteSpace(tok.Literal) - case TYPE_NEWLINE: - return WhiteSpace(tok.Literal) - default: - return Word(tok.Literal) - } - -} - -func (p *Parser) appendElement(b Element) { - p.els = append(p.els, b) -} - -func (p *Parser) consume() bool { - tok, ok := <-p.in - if !ok { - return false - } - p.ts = append(p.ts, tok) - - return true -} - -func (p *Parser) peek() (Token, bool) { - if p.r == len(p.ts) { - if ok := p.consume(); !ok { - return Token{}, false - } - } - - return p.ts[p.r], true -} - -func (p *Parser) readN(count int) ([]Token, bool) { - toks := []Token{} - for i := 0; i < count; i++ { - tok, ok := p.read() - if !ok { - p.unread(len(toks)) - return []Token{}, false - } - toks = append(toks, tok) - } - - return toks, true -} - -func (p *Parser) read() (Token, bool) { - if p.r == len(p.ts) { - if ok := p.consume(); !ok { - return Token{}, false - } - } - - tok := p.ts[p.r] - p.r++ - - return tok, true -} - -func (p *Parser) unread(count int) { - for i := count; i > 0; i-- { - p.r-- - } -} - -func (p *Parser) discard() { - p.ts = p.ts[p.r:] - p.r = 0 -} diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 0000000..eb7c851 --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,101 @@ +package parser + +import ( + "io" + + "ewintr.nl/adoc" + "ewintr.nl/adoc/element" + "ewintr.nl/adoc/token" +) + +type Parser struct { + doc *adoc.ADoc + tr *token.TokenReader + els []element.Element +} + +func New(reader io.Reader) *Parser { + lex := token.NewLexer(reader) + return NewParserFromChannel(lex.Out()) +} + +func NewParserFromChannel(toks chan token.Token) *Parser { + return &Parser{ + doc: adoc.NewADoc(), + tr: token.NewTokenReader(toks), + els: []element.Element{}, + } +} + +func (p *Parser) Parse() *adoc.ADoc { + result, ok := element.NewHeaderFromTokens(p.tr) + if ok { + if h, ok := result.Element.(element.Header); ok { + p.doc.Title = h.Title + p.doc.Author = h.Author + p.doc.Date = h.Date + p.doc.Attributes = h.Attributes + } + } + + for { + if done := p.scan(); done { + break + } + p.tr.Discard() + } + + p.doc.Content = p.els + return p.doc +} + +func (p *Parser) Elements() []element.Element { + return p.els +} + +func (p *Parser) scan() bool { + if _, ok := p.tr.Read(1); !ok { + return true + } + p.tr.Unread(1) + + type tryFunc func(element.ReadUnreader) (element.ParseResult, bool) + for _, tf := range []tryFunc{ + element.NewCodeBlockFromTokens, + element.NewSubTitleFromTokens, + element.NewListFromTokens, + element.NewListItemFromTokens, + element.NewParagraphFromTokens, + element.NewStyleFromTokens, + element.NewLinkFromTokens, + } { + result, ok := tf(p.tr) + if !ok { + continue + } + el := result.Element + if len(result.Inner) != 0 { + par := NewParserFromChannel(token.NewTokenStream(result.Inner).Out()) + par.Parse() + el = el.Append(par.Elements()) + } + p.appendElement(el) + return false + } + + p.doPlain() + + return false +} + +func (p *Parser) doPlain() { + tok, ok := p.tr.Read(1) + if !ok || tok[0].Type == token.TYPE_EOF || tok[0].Type == token.TYPE_EOS { + return + } + p.appendElement(element.MakePlain(tok[0])) +} + +func (p *Parser) appendElement(b element.Element) { + p.els = append(p.els, b) +} diff --git a/parser_test.go b/parser/parser_test.go similarity index 78% rename from parser_test.go rename to parser/parser_test.go index 495a067..293805a 100644 --- a/parser_test.go +++ b/parser/parser_test.go @@ -1,10 +1,11 @@ -package adoc_test +package parser_test import ( "strings" "testing" "ewintr.nl/adoc" + "ewintr.nl/adoc/parser" "ewintr.nl/go-kit/test" ) @@ -20,7 +21,7 @@ func TestParser(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - par := adoc.NewParser(strings.NewReader(tc.input)) + par := parser.New(strings.NewReader(tc.input)) test.Equals(t, tc.exp, par.Parse()) }) } diff --git a/styles.go b/styles.go deleted file mode 100644 index 442ec9c..0000000 --- a/styles.go +++ /dev/null @@ -1,92 +0,0 @@ -package adoc - -type Strong []Element - -func (st Strong) Text() string { - var txt string - for _, e := range st { - txt += e.Text() - } - - return txt -} - -type Emphasis []Element - -func (em Emphasis) Text() string { - var txt string - for _, e := range em { - txt += e.Text() - } - - return txt -} - -type Code []Element - -func (c Code) Text() string { - var txt string - for _, e := range c { - txt += e.Text() - } - - return txt -} - -func (p *Parser) tryStyles() bool { - tok, ok := p.readN(2) - if !ok { - return false - } - markers := []Token{ - {Type: TYPE_ASTERISK, Literal: "*"}, - {Type: TYPE_UNDERSCORE, Literal: "_"}, - {Type: TYPE_BACKTICK, Literal: "`"}, - } - if !tok[0].Equal(markers...) { - p.unread(2) - return false - } - if tok[1].Type == TYPE_WHITESPACE { - p.unread(2) - return false - } - - marker := tok[0] - toks := []Token{tok[1]} - for { - tok, ok := p.read() - if !ok { - p.unread(len(toks) + 1) - return false - } - if tok.Equal(TOKEN_EOF, TOKEN_DOUBLE_NEWLINE) { - p.unread(len(toks) + 1) - return false - } - if tok.Equal(marker) { - break - } - toks = append(toks, tok) - } - if toks[len(toks)-1].Type == TYPE_WHITESPACE { - p.unread(len(toks) + 2) - return false - } - - par := NewParserFromChannel(NewTokenStream(toks).Out()) - par.Parse() - var b Element - switch marker.Type { - case TYPE_ASTERISK: - b = Strong(par.Elements()) - case TYPE_UNDERSCORE: - b = Emphasis(par.Elements()) - case TYPE_BACKTICK: - b = Code(par.Elements()) - } - - p.appendElement(b) - - return true -} diff --git a/styles_test.go b/styles_test.go deleted file mode 100644 index 180bab4..0000000 --- a/styles_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package adoc_test - -import ( - "strings" - "testing" - - "ewintr.nl/adoc" - "ewintr.nl/go-kit/test" -) - -func TestStyles(t *testing.T) { - for _, tc := range []struct { - name string - input string - exp []adoc.Element - }{ - { - name: "strong", - input: "*strong*", - exp: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Strong{adoc.Word("strong")}, - }, - ), - }, - }, - { - name: "emphasis", - input: "_emphasis_", - exp: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Emphasis{adoc.Word("emphasis")}, - }, - ), - }, - }, - { - name: "code", - input: "`code`", - exp: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Code{adoc.Word("code")}, - }, - ), - }, - }, - { - name: "mixed", - input: "some `code code` in plain", - exp: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Word("some"), - adoc.WhiteSpace(" "), - adoc.Code{ - adoc.Word("code"), - adoc.WhiteSpace(" "), - adoc.Word("code"), - }, - adoc.WhiteSpace(" "), - adoc.Word("in"), - adoc.WhiteSpace(" "), - adoc.Word("plain"), - }, - ), - }, - }, - { - name: "incomplete", - input: "a *word", - exp: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Word("a"), - adoc.WhiteSpace(" "), - adoc.Word("*"), - adoc.Word("word"), - }), - }, - }, - { - name: "trailing space", - input: "*word *", - exp: []adoc.Element{ - adoc.Paragraph([]adoc.Element{ - adoc.Word("*"), - adoc.Word("word"), - adoc.WhiteSpace(" "), - adoc.Word("*"), - }), - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - par := adoc.NewParser(strings.NewReader(tc.input)) - exp := &adoc.ADoc{ - Attributes: map[string]string{}, - Content: tc.exp, - } - test.Equals(t, exp, par.Parse()) - }) - } -} diff --git a/subtitle.go b/subtitle.go deleted file mode 100644 index 86d0b33..0000000 --- a/subtitle.go +++ /dev/null @@ -1,52 +0,0 @@ -package adoc - -type SubTitle string - -func (st SubTitle) Text() string { return string(st) } - -type SubSubTitle string - -func (st SubSubTitle) Text() string { return string(st) } - -func (p *Parser) trySubTitle() bool { - toks, ok := p.readN(2) - if !ok { - return false - } - if toks[0].Type != TYPE_EQUALSIGN || toks[0].Len() < 2 || toks[1].Type != TYPE_WHITESPACE { - p.unread(2) - return false - } - - for { - tok, ok := p.read() - if !ok { - p.unread(len(toks)) - return false - } - if (tok.Type == TYPE_NEWLINE && tok.Len() > 1) || tok.Equal(TOKEN_EOF) { - break - } - toks = append(toks, tok) - } - - var title string - for _, tok := range toks[2:] { - if tok.Type == TYPE_NEWLINE { - continue - } - title += p.makePlain(tok).Text() - } - switch toks[0].Len() { - case 2: - p.appendElement(SubTitle(title)) - case 3: - p.appendElement(SubSubTitle(title)) - default: - // ignore lower levels for now - p.unread(len(toks)) - return false - } - - return true -} diff --git a/lexer.go b/token/lexer.go similarity index 98% rename from lexer.go rename to token/lexer.go index a34314b..f41f23c 100644 --- a/lexer.go +++ b/token/lexer.go @@ -1,4 +1,4 @@ -package adoc +package token import ( "bufio" diff --git a/lexer_test.go b/token/lexer_test.go similarity index 77% rename from lexer_test.go rename to token/lexer_test.go index 972f716..b2cd701 100644 --- a/lexer_test.go +++ b/token/lexer_test.go @@ -1,32 +1,32 @@ -package adoc_test +package token_test import ( "strings" "testing" "time" - "ewintr.nl/adoc" + "ewintr.nl/adoc/token" "ewintr.nl/go-kit/test" ) func TestLexer(t *testing.T) { - word := adoc.TYPE_WORD - ws := adoc.TYPE_WHITESPACE - nl := adoc.TYPE_NEWLINE - eq := adoc.TYPE_EQUALSIGN - bt := adoc.TYPE_BACKTICK - as := adoc.TYPE_ASTERISK - un := adoc.TYPE_UNDERSCORE + word := token.TYPE_WORD + ws := token.TYPE_WHITESPACE + nl := token.TYPE_NEWLINE + eq := token.TYPE_EQUALSIGN + bt := token.TYPE_BACKTICK + as := token.TYPE_ASTERISK + un := token.TYPE_UNDERSCORE for _, tc := range []struct { name string input string - exp []adoc.Token + exp []token.Token }{ { name: "word string", input: "one two", - exp: []adoc.Token{ + exp: []token.Token{ {Type: word, Literal: "one"}, {Type: ws, Literal: " "}, {Type: word, Literal: "two"}, @@ -35,7 +35,7 @@ func TestLexer(t *testing.T) { { name: "punctuation", input: `. ,`, - exp: []adoc.Token{ + exp: []token.Token{ {Type: word, Literal: "."}, {Type: ws, Literal: " "}, {Type: word, Literal: ","}, @@ -44,24 +44,24 @@ func TestLexer(t *testing.T) { { name: "whitespace", input: " \t", - exp: []adoc.Token{ + exp: []token.Token{ {Type: ws, Literal: " \t"}, }, }, { name: "tab", input: "\t", - exp: []adoc.Token{{Type: ws, Literal: "\t"}}, + exp: []token.Token{{Type: ws, Literal: "\t"}}, }, { name: "newlines", input: "\n\n\n", - exp: []adoc.Token{{Type: nl, Literal: "\n\n\n"}}, + exp: []token.Token{{Type: nl, Literal: "\n\n\n"}}, }, { name: "special chars", input: "=*_", - exp: []adoc.Token{ + exp: []token.Token{ {Type: eq, Literal: "="}, {Type: as, Literal: "*"}, {Type: un, Literal: "_"}, @@ -70,7 +70,7 @@ func TestLexer(t *testing.T) { { name: "mixed", input: "This is a line with mixed \t `stuff`, see\t==?", - exp: []adoc.Token{ + exp: []token.Token{ {Type: word, Literal: "This"}, {Type: ws, Literal: " "}, {Type: word, Literal: "is"}, @@ -97,8 +97,8 @@ func TestLexer(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { input := strings.NewReader(tc.input) - lex := adoc.NewLexer(input) - act := []adoc.Token{} + lex := token.NewLexer(input) + act := []token.Token{} stop := time.Now().Add(3 * time.Second) T: @@ -118,7 +118,7 @@ func TestLexer(t *testing.T) { } test.OK(t, lex.Error()) - exp := append(tc.exp, adoc.TOKEN_EOF) + exp := append(tc.exp, token.TOKEN_EOF) test.Equals(t, exp, act) }) } diff --git a/token.go b/token/token.go similarity index 99% rename from token.go rename to token/token.go index 454c925..7f41ea2 100644 --- a/token.go +++ b/token/token.go @@ -1,4 +1,4 @@ -package adoc +package token const ( TYPE_EOF TokenType = iota diff --git a/token/tokenreader.go b/token/tokenreader.go new file mode 100644 index 0000000..94d5e2c --- /dev/null +++ b/token/tokenreader.go @@ -0,0 +1,65 @@ +package token + +type TokenReader struct { + in chan Token + ts []Token + r int +} + +func NewTokenReader(in chan Token) *TokenReader { + return &TokenReader{ + in: in, + ts: []Token{}, + } +} + +func (tr *TokenReader) Read(n int) ([]Token, bool) { + toks := []Token{} + for i := 0; i < n; i++ { + tok, ok := tr.readOne() + if !ok { + tr.Unread(len(toks)) + return []Token{}, false + } + toks = append(toks, tok) + } + return toks, true +} + +func (tr *TokenReader) Unread(n int) bool { + //if n > tr.r { + // return false + //} + for i := n; i > 0; i-- { + tr.r-- + } + return true +} + +func (tr *TokenReader) Discard() { + tr.ts = tr.ts[tr.r:] + tr.r = 0 +} + +func (tr *TokenReader) readOne() (Token, bool) { + if tr.r == len(tr.ts) { + if ok := tr.consume(); !ok { + return Token{}, false + } + } + + tok := tr.ts[tr.r] + tr.r++ + + return tok, true +} + +func (tr *TokenReader) consume() bool { + tok, ok := <-tr.in + if !ok { + return false + } + tr.ts = append(tr.ts, tok) + + return true +} diff --git a/tokenstream.go b/token/tokenstream.go similarity index 91% rename from tokenstream.go rename to token/tokenstream.go index fa49d75..06a9567 100644 --- a/tokenstream.go +++ b/token/tokenstream.go @@ -1,4 +1,4 @@ -package adoc +package token type TokenStream struct { ts []Token @@ -20,6 +20,7 @@ func (s *TokenStream) run() { for _, tok := range s.ts { s.out <- tok } + s.out <- TOKEN_EOS close(s.out) }