message formatting and tests

This commit is contained in:
Erik Winter 2021-01-29 17:22:07 +01:00
parent e1758db30e
commit 32fad4414a
8 changed files with 423 additions and 132 deletions

View File

@ -1,7 +1,6 @@
package main
import (
"fmt"
"log"
"os"
@ -10,74 +9,31 @@ import (
)
func main() {
config := &mstore.EmailConfiguration{
IMAPURL: os.Getenv("IMAP_URL"),
IMAPUsername: os.Getenv("IMAP_USERNAME"),
IMAPPassword: os.Getenv("IMAP_PASSWORD"),
config := &mstore.ImapConfiguration{
ImapUrl: os.Getenv("IMAP_URL"),
ImapUsername: os.Getenv("IMAP_USERNAME"),
ImapPassword: os.Getenv("IMAP_PASSWORD"),
}
if !config.Valid() {
log.Fatal("please set MAIL_USER, MAIL_PASSWORD, etc environment variables")
log.Fatal("please set IMAP_USER, IMAP_PASSWORD, etc environment variables")
}
mailStore, err := mstore.EmailConnect(config)
mailStore, err := mstore.ImapConnect(config)
if err != nil {
log.Fatal(err)
}
defer mailStore.Disconnect()
taskRepo := task.NewRepository(mailStore)
tasks, err := taskRepo.FindAll("INBOX")
tasks, err := taskRepo.FindAll(task.FOLDER_INBOX)
if err != nil {
log.Fatal(err)
}
for _, t := range tasks {
fmt.Printf("processing: %s... ", t.Action)
if t.Dirty() {
if t.Dirty {
if err := taskRepo.Update(t); err != nil {
log.Fatal(err)
}
fmt.Printf("updated.")
}
fmt.Printf("\n")
}
/*
folders, err := mailStore.FolderNames()
if err != nil {
log.Fatal(err)
}
for _, f := range folders {
fmt.Println(f)
}
if err := mailStore.Select("Today"); err != nil {
log.Fatal(err)
}
messages, err := mailStore.Messages()
if err != nil {
log.Fatal(err)
}
for _, m := range messages {
fmt.Printf("%d: %s\n", m.Uid, m.Subject)
}
if len(messages) == 0 {
log.Fatal("no messages")
return
}
if err := mailStore.Remove(messages[0].Uid); err != nil {
log.Fatal(err)
}
body := NewBody(`From: todo <process@erikwinter.nl>
Subject: the subject
And here comes the body`)
if err := mailStore.Append("INBOX", imap.Literal(body)); err != nil {
log.Fatal(err)
}
*/
}

1
go.mod
View File

@ -5,6 +5,7 @@ go 1.14
require (
git.sr.ht/~ewintr/go-kit v0.0.0-20201229104230-4d7958f8de04
github.com/emersion/go-imap v1.0.6
github.com/emersion/go-message v0.11.1
github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect
github.com/google/uuid v1.2.0
)

3
go.sum
View File

@ -48,11 +48,13 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP
github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
github.com/emersion/go-imap v1.0.6 h1:N9+o5laOGuntStBo+BOgfEB5evPsPD+K5+M0T2dctIc=
github.com/emersion/go-imap v1.0.6/go.mod h1:yKASt+C3ZiDAiCSssxg9caIckWF/JG7ZQTO7GAmvicU=
github.com/emersion/go-message v0.11.1 h1:0C/S4JIXDTSfXB1vpqdimAYyK4+79fgEAMQ0dSL+Kac=
github.com/emersion/go-message v0.11.1/go.mod h1:C4jnca5HOTo4bGN9YdqNQM9sITuT3Y0K6bSUw9RklvY=
github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b h1:uhWtEWBHgop1rqEk2klKaxPAkVDCXexai6hSuRQ7Nvs=
github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k=
github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ=
github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe h1:40SWqY0zE3qCi6ZrtTf5OUdNm5lDnGnjRSq9GgmeTrg=
github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@ -141,6 +143,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/martinlindhe/base36 v1.0.0 h1:eYsumTah144C0A8P1T/AVSUk5ZoLnhfYFM3OGQxB52A=
github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=

9
internal/task/date.go Normal file
View File

@ -0,0 +1,9 @@
package task
import "time"
type Date time.Time
func (d *Date) Weekday() Weekday {
return d.Weekday()
}

View File

@ -30,7 +30,7 @@ func (tr *TaskRepo) FindAll(folder string) ([]*Task, error) {
tasks := []*Task{}
for _, msg := range msgs {
if msg.Valid() {
tasks = append(tasks, NewFromMessage(msg))
tasks = append(tasks, New(msg))
}
}
@ -41,12 +41,12 @@ func (tr *TaskRepo) Update(t *Task) error {
if !t.Current {
return ErrOutdatedTask
}
if !t.Dirty() {
if !t.Dirty {
return nil
}
// add new
if err := tr.mstore.Add(t.Folder, t.Subject(), t.Body()); err != nil {
if err := tr.mstore.Add(t.Folder, t.FormatSubject(), t.FormatBody()); err != nil {
return fmt.Errorf("%w: %s", ErrMStoreError, err)
}

View File

@ -4,7 +4,6 @@ import (
"errors"
"fmt"
"strings"
"time"
"git.sr.ht/~ewintr/gte/pkg/mstore"
"github.com/google/uuid"
@ -14,37 +13,61 @@ var (
ErrOutdatedTask = errors.New("task is outdated")
)
type Date time.Time
const (
FOLDER_INBOX = "INBOX"
FOLDER_NEW = "New"
func (d *Date) Weekday() Weekday {
return d.Weekday()
}
FIELD_SEPARATOR = ":"
FIELD_ID = "id"
FIELD_ACTION = "action"
)
// Task reperesents a task based on the data stored in a message
type Task struct {
// Id is a UUID that gets carried over when a new message is constructed
Id string
// Folder is the same name as the mstore folder
Folder string
Action string
Due Date
Message *mstore.Message
// Current indicates whether the task represents an existing message in the mstore
Current bool
Simplified bool
// Dirty indicates whether the task contains updates not present in the message
Dirty bool
}
func NewFromMessage(msg *mstore.Message) *Task {
fmt.Println(msg.Subject)
id := FieldFromBody("id", msg.Body)
// New constructs a Task based on an mstore.Message.
//
// The data in the message is stored as key: value pairs, one per line. The line can start with quoting marks.
// The subject line also contains values in the format "date - project - action".
// Keys that exist more than once are merged. The one that appears first in the body takes precedence. A value present in the Body takes precedence over one in the subject.
// This enables updating a task by forwarding a topposted message whith new values for fields that the user wants to update.
func New(msg *mstore.Message) *Task {
dirty := false
id := FieldFromBody(FIELD_ID, msg.Body)
if id == "" {
id = uuid.New().String()
dirty = true
}
action := FieldFromBody("action", msg.Body)
action := FieldFromBody(FIELD_ACTION, msg.Body)
if action == "" {
action = FieldFromSubject("action", msg.Subject)
action = FieldFromSubject(FIELD_ACTION, msg.Subject)
if action != "" {
dirty = true
}
}
folder := msg.Folder
if folder == "INBOX" {
folder = "New"
if folder == FOLDER_INBOX {
folder = FOLDER_NEW
dirty = true
}
return &Task{
@ -53,40 +76,37 @@ func NewFromMessage(msg *mstore.Message) *Task {
Folder: folder,
Message: msg,
Current: true,
Simplified: false,
Dirty: dirty,
}
}
// Dirty checks if the task has unsaved changes
func (t *Task) Dirty() bool {
mBody := t.Message.Body
mSubject := t.Message.Subject
if t.Id != FieldFromBody("id", mBody) {
return true
}
if t.Folder != t.Message.Folder {
return true
}
if t.Action != FieldFromBody("action", mBody) {
return true
}
if t.Action != FieldFromSubject("action", mSubject) {
return true
}
return false
}
func (t *Task) Subject() string {
func (t *Task) FormatSubject() string {
return t.Action
}
func (t *Task) Body() string {
body := fmt.Sprintf("id: %s\n", t.Id)
body += fmt.Sprintf("action: %s\n", t.Action)
func (t *Task) FormatBody() string {
body := fmt.Sprintf("\n")
order := []string{FIELD_ID, FIELD_ACTION}
fields := map[string]string{
FIELD_ID: t.Id,
FIELD_ACTION: t.Action,
}
keyLen := 0
for _, f := range order {
if len(f) > keyLen {
keyLen = len(f)
}
}
for _, f := range order {
key := f + FIELD_SEPARATOR
for i := len(key); i <= keyLen; i++ {
key += " "
}
line := strings.TrimSpace(fmt.Sprintf("%s %s", key, fields[f]))
body += fmt.Sprintf("%s\n", line)
}
return body
}
@ -94,11 +114,11 @@ func (t *Task) Body() string {
func FieldFromBody(field, body string) string {
lines := strings.Split(body, "\n")
for _, line := range lines {
parts := strings.SplitN(line, ":", 2)
parts := strings.SplitN(line, FIELD_SEPARATOR, 2)
if len(parts) < 2 {
continue
}
if strings.ToLower(parts[0]) == field {
if strings.ToLower(strings.TrimSpace(parts[0])) == field {
return strings.TrimSpace(parts[1])
}
}
@ -107,7 +127,7 @@ func FieldFromBody(field, body string) string {
}
func FieldFromSubject(field, subject string) string {
if field == "action" {
if field == FIELD_ACTION {
return strings.ToLower(subject)
}

260
internal/task/task_test.go Normal file
View File

@ -0,0 +1,260 @@
package task_test
import (
"fmt"
"testing"
"git.sr.ht/~ewintr/go-kit/test"
"git.sr.ht/~ewintr/gte/internal/task"
"git.sr.ht/~ewintr/gte/pkg/mstore"
)
func TestNewFromMessage(t *testing.T) {
id := "an id"
action := "some action"
folder := task.FOLDER_NEW
for _, tc := range []struct {
name string
message *mstore.Message
hasId bool
exp *task.Task
}{
{
name: "empty",
message: &mstore.Message{},
exp: &task.Task{
Dirty: true,
},
},
{
name: "with id, action and folder",
message: &mstore.Message{
Folder: folder,
Body: fmt.Sprintf(`
id: %s
action: %s
`, id, action),
},
hasId: true,
exp: &task.Task{
Id: id,
Folder: folder,
Action: action,
},
},
{
name: "folder inbox get updated to new",
message: &mstore.Message{
Folder: task.FOLDER_INBOX,
Body: fmt.Sprintf(`
id: %s
action: %s
`, id, action),
},
hasId: true,
exp: &task.Task{
Id: id,
Folder: task.FOLDER_NEW,
Action: action,
Dirty: true,
},
},
{
name: "action in subject takes precedence",
message: &mstore.Message{
Folder: folder,
Subject: "some other action",
Body: fmt.Sprintf(`
id: %s
action: %s
`, id, action),
},
exp: &task.Task{
Id: id,
Folder: folder,
Action: action,
},
},
{
name: "action from subject if not present in body",
message: &mstore.Message{
Folder: folder,
Subject: action,
Body: fmt.Sprintf(`id: %s`, id),
},
exp: &task.Task{
Id: id,
Folder: folder,
Action: action,
Dirty: true,
},
},
} {
t.Run(tc.name, func(t *testing.T) {
act := task.New(tc.message)
if !tc.hasId {
test.Equals(t, false, "" == act.Id)
tc.exp.Id = act.Id
}
tc.exp.Message = tc.message
tc.exp.Current = true
test.Equals(t, tc.exp, act)
})
}
}
func TestFormatSubject(t *testing.T) {
action := "an action"
for _, tc := range []struct {
name string
task *task.Task
exp string
}{
{
name: "empty",
task: &task.Task{},
},
{
name: "with action",
task: &task.Task{Action: action},
exp: action,
},
} {
t.Run(tc.name, func(t *testing.T) {
test.Equals(t, tc.exp, tc.task.FormatSubject())
})
}
}
func TestFormatBody(t *testing.T) {
id := "an id"
action := "an action"
for _, tc := range []struct {
name string
task *task.Task
exp string
}{
{
name: "empty",
task: &task.Task{},
exp: `
id:
action:
`,
},
{
name: "filled",
task: &task.Task{
Id: id,
Action: action,
},
exp: `
id: an id
action: an action
`,
},
} {
t.Run(tc.name, func(t *testing.T) {
test.Equals(t, tc.exp, tc.task.FormatBody())
})
}
}
func TestFieldFromBody(t *testing.T) {
for _, tc := range []struct {
name string
field string
body string
exp string
}{
{
name: "empty field",
body: `field: value`,
},
{
name: "empty body",
field: "field",
},
{
name: "not present",
field: "field",
body: "another: value",
},
{
name: "present",
field: "fieldb",
body: `
not a field at all
fielda: valuea
fieldb: valueb
fieldc: valuec
`,
exp: "valueb",
},
{
name: "present twice",
field: "field",
body: `
field: valuea
field: valueb
`,
exp: "valuea",
},
{
name: "with colons",
field: "field",
body: "field:: val:ue",
exp: ": val:ue",
},
{
name: "trim field",
field: "field",
body: " field : value",
exp: "value",
},
{
name: "trim value",
field: "field",
body: "field: value ",
exp: "value",
},
} {
t.Run(tc.name, func(t *testing.T) {
test.Equals(t, tc.exp, task.FieldFromBody(tc.field, tc.body))
})
}
}
func TestFieldFromSubject(t *testing.T) {
for _, tc := range []struct {
name string
field string
subject string
exp string
}{
{
name: "empty field",
subject: "subject",
},
{
name: "empty subject",
field: task.FIELD_ACTION,
},
{
name: "unknown field",
field: "unknown",
subject: "subject",
},
{
name: "known field",
field: task.FIELD_ACTION,
subject: "subject",
exp: "subject",
},
} {
t.Run(tc.name, func(t *testing.T) {
test.Equals(t, tc.exp, task.FieldFromSubject(tc.field, tc.subject))
})
}
}

View File

@ -3,11 +3,13 @@ package mstore
import (
"fmt"
"io"
"io/ioutil"
"strings"
"time"
"github.com/emersion/go-imap"
"github.com/emersion/go-imap/client"
"github.com/emersion/go-message/mail"
)
type Body struct {
@ -31,47 +33,47 @@ func (b *Body) Len() int {
return b.length
}
type EmailConfiguration struct {
IMAPURL string
IMAPUsername string
IMAPPassword string
type ImapConfiguration struct {
ImapUrl string
ImapUsername string
ImapPassword string
}
func (esc *EmailConfiguration) Valid() bool {
if esc.IMAPURL == "" {
func (esc *ImapConfiguration) Valid() bool {
if esc.ImapUrl == "" {
return false
}
if esc.IMAPUsername == "" || esc.IMAPPassword == "" {
if esc.ImapUsername == "" || esc.ImapPassword == "" {
return false
}
return true
}
type Email struct {
type Imap struct {
imap *client.Client
mboxStatus *imap.MailboxStatus
}
func EmailConnect(conf *EmailConfiguration) (*Email, error) {
imap, err := client.DialTLS(conf.IMAPURL, nil)
func ImapConnect(conf *ImapConfiguration) (*Imap, error) {
imap, err := client.DialTLS(conf.ImapUrl, nil)
if err != nil {
return &Email{}, err
return &Imap{}, err
}
if err := imap.Login(conf.IMAPUsername, conf.IMAPPassword); err != nil {
return &Email{}, err
if err := imap.Login(conf.ImapUsername, conf.ImapPassword); err != nil {
return &Imap{}, err
}
return &Email{
return &Imap{
imap: imap,
}, nil
}
func (es *Email) Disconnect() {
func (es *Imap) Disconnect() {
es.imap.Logout()
}
func (es *Email) Folders() ([]string, error) {
func (es *Imap) Folders() ([]string, error) {
boxes, done := make(chan *imap.MailboxInfo), make(chan error)
go func() {
done <- es.imap.List("", "*", boxes)
@ -89,7 +91,7 @@ func (es *Email) Folders() ([]string, error) {
return folders, nil
}
func (es *Email) selectFolder(folder string) error {
func (es *Imap) selectFolder(folder string) error {
status, err := es.imap.Select(folder, false)
if err != nil {
return err
@ -100,7 +102,7 @@ func (es *Email) selectFolder(folder string) error {
return nil
}
func (es *Email) Messages(folder string) ([]*Message, error) {
func (es *Imap) Messages(folder string) ([]*Message, error) {
if err := es.selectFolder(folder); err != nil {
return []*Message{}, err
}
@ -112,17 +114,57 @@ func (es *Email) Messages(folder string) ([]*Message, error) {
seqset := new(imap.SeqSet)
seqset.AddRange(uint32(1), es.mboxStatus.Messages)
// Get the whole message body
section := &imap.BodySectionName{}
items := []imap.FetchItem{imap.FetchUid, section.FetchItem()}
imsg, done := make(chan *imap.Message), make(chan error)
go func() {
done <- es.imap.Fetch(seqset, []imap.FetchItem{imap.FetchEnvelope, imap.FetchUid}, imsg)
done <- es.imap.Fetch(seqset, items, imsg)
}()
messages := []*Message{}
for m := range imsg {
r := m.GetBody(section)
if r == nil {
return []*Message{}, fmt.Errorf("server didn't returned message body")
}
// Create a new mail reader
mr, err := mail.CreateReader(r)
if err != nil {
return []*Message{}, err
}
// Print some info about the message
header := mr.Header
subject, err := header.Subject()
if err != nil {
return []*Message{}, err
}
// Process each message's part
body := []byte(``)
for {
p, err := mr.NextPart()
if err == io.EOF {
break
} else if err != nil {
return []*Message{}, err
}
switch p.Header.(type) {
case *mail.InlineHeader:
// This is the message's text (can be plain-text or HTML)
body, _ = ioutil.ReadAll(p.Body)
}
}
messages = append(messages, &Message{
Uid: m.Uid,
Folder: folder,
Subject: m.Envelope.Subject,
Subject: subject,
Body: string(body),
})
}
@ -133,7 +175,7 @@ func (es *Email) Messages(folder string) ([]*Message, error) {
return messages, nil
}
func (es *Email) Add(folder, subject, body string) error {
func (es *Imap) Add(folder, subject, body string) error {
msgStr := fmt.Sprintf(`From: todo <process@erikwinter.nl>
Subject: %s
@ -144,7 +186,7 @@ Subject: %s
return es.imap.Append(folder, nil, time.Time{}, imap.Literal(msg))
}
func (es *Email) Remove(msg *Message) error {
func (es *Imap) Remove(msg *Message) error {
if !msg.Valid() {
return ErrInvalidMessage
}