basic feed fetch

feed/miniflux.go

@@ -1,47 +0,0 @@
-package feed
-
-import (
-	"fmt"
-	"miniflux.app/client"
-)
-
-type Miniflux struct {
-	client *client.Client
-}
-
-func NewMiniflux(url, apiKey string) *Miniflux {
-	return &Miniflux{
-		client: client.New(url, apiKey),
-	}
-}
-
-func (m *Miniflux) Feeds() error {
-
-	feeds, err := m.client.Feeds()
-	if err != nil {
-		fmt.Println(err)
-		return err
-	}
-	fmt.Println(feeds)
-
-	return nil
-}
-
-type Entry struct {
-	ChannelID int
-	Title     string
-	URL       string
-}
-
-func (m *Miniflux) Unread() ([]Entry, error) {
-	result, err := m.client.Entries(&client.Filter{Status: "unread"})
-	if err != nil {
-		return []Entry{}, err
-	}
-
-	for _, entry := range result.Entries {
-		fmt.Println(entry.ID, entry.Title, entry.URL)
-	}
-
-	return []Entry{}, nil
-}

fetch/fetch.go

@@ -0,0 +1,56 @@
+package fetch
+
+import (
+	"ewintr.nl/yogai/model"
+	"ewintr.nl/yogai/storage"
+	"log"
+	"time"
+)
+
+type FeedReader interface {
+	Unread() ([]*model.Video, error)
+	MarkRead(feedID string) error
+}
+
+type Fetch struct {
+	interval   time.Duration
+	videoRepo  storage.VideoRepository
+	feedReader FeedReader
+	out        chan<- model.Video
+}
+
+func NewFetch(videoRepo storage.VideoRepository, feedReader FeedReader, interval time.Duration) *Fetch {
+	return &Fetch{
+		interval:   interval,
+		videoRepo:  videoRepo,
+		feedReader: feedReader,
+	}
+}
+
+func (v *Fetch) Run() {
+	ticker := time.NewTicker(v.interval)
+	for {
+		select {
+		case <-ticker.C:
+			newVideos, err := v.feedReader.Unread()
+			if err != nil {
+				log.Println(err)
+			}
+			for _, video := range newVideos {
+				if err := v.videoRepo.Save(video); err != nil {
+					log.Println(err)
+					continue
+				}
+				//v.out <- video
+				if err := v.feedReader.MarkRead(video.FeedID); err != nil {
+					log.Println(err)
+				}
+			}
+
+		}
+	}
+}
+
+func (v *Fetch) Out() chan<- model.Video {
+	return v.out
+}

fetch/miniflux.go

@@ -0,0 +1,65 @@
+package fetch
+
+import (
+	"ewintr.nl/yogai/model"
+	"github.com/google/uuid"
+	"miniflux.app/client"
+	"strconv"
+)
+
+type Entry struct {
+	MinifluxEntryID string
+	MinifluxFeedID  string
+	MinifluxURL     string
+	Title           string
+	Description     string
+}
+
+type MinifluxInfo struct {
+	Endpoint string
+	ApiKey   string
+}
+
+type Miniflux struct {
+	client *client.Client
+}
+
+func NewMiniflux(mflInfo MinifluxInfo) *Miniflux {
+	return &Miniflux{
+		client: client.New(mflInfo.Endpoint, mflInfo.ApiKey),
+	}
+}
+
+func (m *Miniflux) Unread() ([]*model.Video, error) {
+	result, err := m.client.Entries(&client.Filter{Status: "unread"})
+	if err != nil {
+		return []*model.Video{}, err
+	}
+
+	videos := []*model.Video{}
+	for _, entry := range result.Entries {
+		videos = append(videos, &model.Video{
+			ID:          uuid.New(),
+			Status:      model.STATUS_NEW,
+			YoutubeURL:  entry.URL,
+			FeedID:      strconv.Itoa(int(entry.ID)),
+			Title:       entry.Title,
+			Description: entry.Content,
+		})
+	}
+
+	return videos, nil
+}
+
+func (m *Miniflux) MarkRead(entryID string) error {
+	id, err := strconv.ParseInt(entryID, 10, 64)
+	if err != nil {
+		return err
+	}
+
+	if err := m.client.UpdateEntries([]int64{id}, "read"); err != nil {
+		return err
+	}
+
+	return nil
+}

go.mod

@@ -3,6 +3,7 @@ module ewintr.nl/yogai
 go 1.20
 
 require (
+	github.com/google/uuid v1.3.0
 	github.com/lib/pq v1.10.9
 	miniflux.app v0.0.0-20230505000442-88062ab9f959
 )

go.sum

@@ -1,3 +1,5 @@
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 miniflux.app v0.0.0-20230505000442-88062ab9f959 h1:YzOQqdFtI6HYRmz8+xVZbqcrVoaC3X4x/pB2DDID//c=

model/video.go

@@ -0,0 +1,21 @@
+package model
+
+import "github.com/google/uuid"
+
+type Status string
+
+const (
+	STATUS_NEW           Status = "new"
+	STATUS_NEEDS_SUMMARY Status = "needs_summary"
+	STATUS_READY         Status = "ready"
+)
+
+type Video struct {
+	ID          uuid.UUID
+	Status      Status
+	YoutubeURL  string
+	FeedID      string
+	Title       string
+	Description string
+	Summary     string
+}

service.go

@@ -1,49 +1,47 @@
 package main
 
 import (
-	"database/sql"
+	"ewintr.nl/yogai/fetch"
-	"ewintr.nl/yogai/feed"
 	"ewintr.nl/yogai/storage"
 	"fmt"
 	"os"
+	"os/signal"
+	"time"
 )
 
 func main() {
-	pgInfo := struct {
+	postgres, err := storage.NewPostgres(storage.PostgresInfo{
-		Host     string
-		Port     string
-		User     string
-		Password string
-		Database string
-	}{
 		Host:     getParam("POSTGRES_HOST", "localhost"),
 		Port:     getParam("POSTGRES_PORT", "5432"),
 		User:     getParam("POSTGRES_USER", "yogai"),
 		Password: getParam("POSTGRES_PASSWORD", "yogai"),
 		Database: getParam("POSTGRES_DB", "yogai"),
-	}
+	})
-	db, err := sql.Open("postgres", fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
-		pgInfo.Host, pgInfo.Port, pgInfo.User, pgInfo.Password, pgInfo.Database))
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}
-	_, err = storage.NewPostgres(db)
+	videoRepo := storage.NewPostgresVideoRepository(postgres)
+
+	mflx := fetch.NewMiniflux(fetch.MinifluxInfo{
+		Endpoint: getParam("MINIFLUX_ENDPOINT", "http://localhost/v1"),
+		ApiKey:   getParam("MINIFLUX_APIKEY", ""),
+	})
+
+	fetchInterval, err := time.ParseDuration(getParam("FETCH_INTERVAL", "1m"))
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}
 
-	mlxInfo := struct {
+	fetcher := fetch.NewFetch(videoRepo, mflx, fetchInterval)
-		Endpoint string
+	go fetcher.Run()
-		ApiKey   string
+
-	}{
+	done := make(chan os.Signal)
-		Endpoint: getParam("MINIFLUX_ENDPOINT", "http://localhost/v1"),
+	signal.Notify(done, os.Interrupt)
-		ApiKey:   getParam("MINIFLUX_APIKEY", ""),
+	<-done
-	}
+
-	mflx := feed.NewMiniflux(mlxInfo.Endpoint, mlxInfo.ApiKey)
+	//fmt.Println(err)
-	_, err = mflx.Unread()
-	fmt.Println(err)
 }
 
 func getParam(param, def string) string {

storage/postgres.go

@@ -2,15 +2,29 @@ package storage
 
 import (
 	"database/sql"
+	"ewintr.nl/yogai/model"
 	"fmt"
 	_ "github.com/lib/pq"
 )
 
+type PostgresInfo struct {
+	Host     string
+	Port     string
+	User     string
+	Password string
+	Database string
+}
+
 type Postgres struct {
 	db *sql.DB
 }
 
-func NewPostgres(db *sql.DB) (*Postgres, error) {
+func NewPostgres(pgInfo PostgresInfo) (*Postgres, error) {
+	db, err := sql.Open("postgres", fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
+		pgInfo.Host, pgInfo.Port, pgInfo.User, pgInfo.Password, pgInfo.Database))
+	if err != nil {
+		return &Postgres{}, err
+	}
 	p := &Postgres{db: db}
 	if err := p.migrate(pgMigration); err != nil {
 		return &Postgres{}, err
@@ -19,18 +33,38 @@ func NewPostgres(db *sql.DB) (*Postgres, error) {
 	return p, nil
 }
 
+type PostgresVideoRepository struct {
+	*Postgres
+}
+
+func NewPostgresVideoRepository(postgres *Postgres) *PostgresVideoRepository {
+	return &PostgresVideoRepository{postgres}
+}
+
+func (p *PostgresVideoRepository) Save(v *model.Video) error {
+	query := `INSERT INTO video (id, status, youtube_url, feed_id, title, description)
+VALUES ($1, $2, $3, $4, $5, $6)
+ON CONFLICT (id)
+DO UPDATE SET
+  id = EXCLUDED.id,
+  status = EXCLUDED.status,
+  youtube_url = EXCLUDED.youtube_url,
+  feed_id = EXCLUDED.feed_id,
+  title = EXCLUDED.title,
+  description = EXCLUDED.description;`
+	_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeURL, v.FeedID, v.Title, v.Description)
+
+	return err
+}
+
 var pgMigration = []string{
-	`CREATE TABLE channel (
+	`CREATE TYPE video_status AS ENUM ('new', 'needs_summary', 'ready')`,
-    id SERIAL PRIMARY KEY,
-    url VARCHAR(255) NOT NULL,
-    feed_url VARCHAR(255) NOT NULL,
-    name VARCHAR(255) NOT NULL
-)`,
 	`CREATE TABLE video (
-    id SERIAL PRIMARY KEY,
+    id uuid PRIMARY KEY,
-    channel_id INTEGER REFERENCES channel(id) ON DELETE CASCADE,
+    status video_status NOT NULL,
-    url VARCHAR(255) NOT NULL,
+    youtube_url VARCHAR(255) NOT NULL,
     title VARCHAR(255) NOT NULL,
+    feed_id VARCHAR(255) NOT NULL,
     description TEXT,
     summary TEXT
 )`,

storage/storage.go

@@ -0,0 +1,9 @@
+package storage
+
+import (
+	"ewintr.nl/yogai/model"
+)
+
+type VideoRepository interface {
+	Save(video *model.Video) error
+}