fix fetch to process

vec model and fix log
multiple pipelines
2023-07-06 14:30:37 +02:00 · 2023-07-06 13:58:12 +02:00 · 2023-07-06 13:37:55 +02:00 · 2023-07-06 13:25:51 +02:00 · 2023-07-04 19:56:44 +02:00 · 2023-05-31 16:27:35 +02:00
27 changed files with 1174 additions and 334 deletions
--- a/11
+++ b/11
@ -0,0 +1,11 @@
+FROM golang:1.20-alpine as build
+
+WORKDIR /src
+COPY . ./
+RUN go mod download
+RUN go build -o /yogai ./service.go
+
+FROM golang:1.20-alpine
+
+COPY --from=build /yogai /yogai
+CMD /yogai
--- a/4
+++ b/4
@ -0,0 +1,4 @@
+docker-push:
+	docker build . -t yogai
+	docker tag yogai registry.ewintr.nl/yogai
+	docker push registry.ewintr.nl/yogai
--- a/fetch/feedreader.go
+++ b/fetch/feedreader.go
@ -0,0 +1,19 @@
+package fetch
+
+import "ewintr.nl/yogai/model"
+
+type FeedEntry struct {
+	EntryID          int64
+	FeedID           int64
+	YoutubeChannelID string
+	YoutubeID        string
+}
+
+type ChannelReader interface {
+	Search(channelID model.YoutubeChannelID, pageToken string) ([]model.YoutubeVideoID, string, error)
+}
+
+type FeedReader interface {
+	Unread() ([]FeedEntry, error)
+	MarkRead(feedID int64) error
+}
--- a/fetch/fetcher.go
+++ b/fetch/fetcher.go
@ -0,0 +1,235 @@
+package fetch
+
+import (
+	"time"
+
+	"ewintr.nl/yogai/model"
+	"ewintr.nl/yogai/storage"
+	"github.com/google/uuid"
+	"golang.org/x/exp/slog"
+)
+
+type Fetcher struct {
+	interval        time.Duration
+	feedRepo        storage.FeedRelRepository
+	videoRepo       storage.VideoRelRepository
+	feedReader      FeedReader
+	channelReader   ChannelReader
+	metadataFetcher MetadataFetcher
+	feedPipeline    chan *model.Feed
+	videoPipeline   chan *model.Video
+	needsMetadata   chan *model.Video
+	out             chan *model.Video
+	logger          *slog.Logger
+}
+
+func NewFetch(feedRepo storage.FeedRelRepository, videoRepo storage.VideoRelRepository, channelReader ChannelReader, feedReader FeedReader, interval time.Duration, metadataFetcher MetadataFetcher, logger *slog.Logger) *Fetcher {
+	return &Fetcher{
+		interval:        interval,
+		feedRepo:        feedRepo,
+		videoRepo:       videoRepo,
+		channelReader:   channelReader,
+		feedReader:      feedReader,
+		metadataFetcher: metadataFetcher,
+		feedPipeline:    make(chan *model.Feed, 10),
+		videoPipeline:   make(chan *model.Video, 10),
+		needsMetadata:   make(chan *model.Video, 10),
+		out:             make(chan *model.Video),
+		logger:          logger,
+	}
+}
+
+func (f *Fetcher) Run() {
+	go f.FetchHistoricalVideos()
+	go f.FindNewFeeds()
+
+	go f.ReadFeeds()
+	go f.MetadataFetcher()
+	go f.FindUnprocessed()
+
+	f.logger.Info("started videoPipeline")
+	for {
+		select {
+		case video := <-f.videoPipeline:
+			if err := f.videoRepo.Save(video); err != nil {
+				f.logger.Error("failed to save video in normal db", err)
+				continue
+			}
+			switch video.Status {
+			case model.StatusNew:
+				f.needsMetadata <- video
+			case model.StatusFetched:
+				f.out <- video
+			}
+		}
+	}
+}
+
+func (f *Fetcher) Out() chan *model.Video {
+	return f.out
+}
+
+func (f *Fetcher) FindNewFeeds() {
+	f.logger.Info("looking for new feeds")
+	feeds, err := f.feedRepo.FindByStatus(model.FeedStatusNew)
+	if err != nil {
+		f.logger.Error("failed to fetch feeds", err)
+		return
+	}
+	for _, feed := range feeds {
+		f.feedPipeline <- feed
+	}
+}
+
+func (f *Fetcher) FetchHistoricalVideos() {
+	f.logger.Info("started historical video fetch")
+
+	for feed := range f.feedPipeline {
+		f.logger.Info("fetching historical videos", slog.String("channelid", string(feed.YoutubeChannelID)))
+		token := ""
+		for {
+			token = f.FetchHistoricalVideoPage(feed.YoutubeChannelID, token)
+			if token == "" {
+				break
+			}
+		}
+		feed.Status = model.FeedStatusReady
+		if err := f.feedRepo.Save(feed); err != nil {
+			f.logger.Error("failed to save feed", err)
+			continue
+		}
+	}
+}
+
+func (f *Fetcher) FetchHistoricalVideoPage(channelID model.YoutubeChannelID, pageToken string) string {
+	f.logger.Info("fetching historical video page", slog.String("channelid", string(channelID)), slog.String("pagetoken", pageToken))
+	ytIDs, pageToken, err := f.channelReader.Search(channelID, pageToken)
+	if err != nil {
+		f.logger.Error("failed to fetch channel", err)
+		return ""
+	}
+	for _, ytID := range ytIDs {
+		video := &model.Video{
+			ID:               uuid.New(),
+			Status:           model.StatusNew,
+			YoutubeID:        ytID,
+			YoutubeChannelID: channelID,
+		}
+		if err := f.videoRepo.Save(video); err != nil {
+			f.logger.Error("failed to save video", err)
+			continue
+		}
+		f.videoPipeline <- video
+	}
+
+	f.logger.Info("fetched historical video page", slog.String("channelid", string(channelID)), slog.String("pagetoken", pageToken), slog.Int("count", len(ytIDs)))
+	return pageToken
+}
+
+func (f *Fetcher) FindUnprocessed() {
+	f.logger.Info("looking for unprocessed videos")
+	videos, err := f.videoRepo.FindByStatus(model.StatusNew, model.StatusFetched)
+	if err != nil {
+		f.logger.Error("failed to fetch unprocessed videos", err)
+		return
+	}
+	f.logger.Info("found unprocessed videos", slog.Int("count", len(videos)))
+	for _, video := range videos {
+		f.videoPipeline <- video
+	}
+}
+
+func (f *Fetcher) ReadFeeds() {
+	f.logger.Info("started feed reader")
+	ticker := time.NewTicker(f.interval)
+	for range ticker.C {
+		entries, err := f.feedReader.Unread()
+		if err != nil {
+			f.logger.Error("failed to fetch unread entries", err)
+			continue
+		}
+		f.logger.Info("fetched unread entries", slog.Int("count", len(entries)))
+		if len(entries) == 0 {
+			continue
+		}
+
+		for _, entry := range entries {
+			video := &model.Video{
+				ID:               uuid.New(),
+				Status:           model.StatusNew,
+				YoutubeID:        model.YoutubeVideoID(entry.YoutubeID),
+				YoutubeChannelID: model.YoutubeChannelID(entry.YoutubeChannelID),
+			}
+			if err := f.videoRepo.Save(video); err != nil {
+				f.logger.Error("failed to save video", err)
+				continue
+			}
+			f.videoPipeline <- video
+			if err := f.feedReader.MarkRead(entry.EntryID); err != nil {
+				f.logger.Error("failed to mark entry as read", err)
+				continue
+			}
+		}
+	}
+}
+
+func (f *Fetcher) MetadataFetcher() {
+	f.logger.Info("started metadata fetch")
+
+	buffer := []*model.Video{}
+	timeout := time.NewTimer(10 * time.Second)
+	fetch := make(chan []*model.Video)
+
+	go func() {
+		for videos := range fetch {
+			f.logger.Info("fetching metadata", slog.Int("count", len(videos)))
+			ids := make([]model.YoutubeVideoID, 0, len(videos))
+			for _, video := range videos {
+				ids = append(ids, video.YoutubeID)
+			}
+			mds, err := f.metadataFetcher.FetchMetadata(ids)
+			if err != nil {
+				f.logger.Error("failed to fetch metadata", err)
+				continue
+			}
+			for _, video := range videos {
+				md := mds[video.YoutubeID]
+				video.YoutubeTitle = md.Title
+				video.YoutubeDescription = md.Description
+				video.YoutubeDuration = md.Duration
+				video.YoutubePublishedAt = md.PublishedAt
+				video.Status = model.StatusFetched
+
+				if err := f.videoRepo.Save(video); err != nil {
+					f.logger.Error("failed to save video", err)
+					continue
+				}
+
+				f.out <- video
+			}
+			f.logger.Info("fetched metadata", slog.Int("count", len(videos)))
+		}
+	}()
+
+	for {
+		select {
+		case video := <-f.needsMetadata:
+			timeout.Reset(10 * time.Second)
+			buffer = append(buffer, video)
+			if len(buffer) >= 50 {
+				batch := make([]*model.Video, len(buffer))
+				copy(batch, buffer)
+				fetch <- batch
+				buffer = []*model.Video{}
+			}
+		case <-timeout.C:
+			if len(buffer) == 0 {
+				continue
+			}
+			batch := make([]*model.Video, len(buffer))
+			copy(batch, buffer)
+			fetch <- batch
+			buffer = []*model.Video{}
+		}
+	}
+}
--- a/fetch/metadata.go
+++ b/fetch/metadata.go
@ -0,0 +1,14 @@
+package fetch
+
+import "ewintr.nl/yogai/model"
+
+type Metadata struct {
+	Title       string
+	Description string
+	Duration    string
+	PublishedAt string
+}
+
+type MetadataFetcher interface {
+	FetchMetadata([]model.YoutubeVideoID) (map[model.YoutubeVideoID]Metadata, error)
+}
--- a/fetcher/miniflux.go
+++ b/fetcher/miniflux.go
@ -1,8 +1,9 @@
-package fetcher
+package fetch

 import (
-	"miniflux.app/client"
 	"strings"
+
+	"miniflux.app/client"
 )

 type Entry struct {
@ -31,24 +32,17 @@ func NewMiniflux(mflInfo MinifluxInfo) *Miniflux {
 func (m *Miniflux) Unread() ([]FeedEntry, error) {
 	result, err := m.client.Entries(&client.Filter{Status: "unread"})
 	if err != nil {
-		return []FeedEntry{}, err
+		return nil, err
 	}

-	entries := []FeedEntry{}
+	entries := make([]FeedEntry, 0, len(result.Entries))
 	for _, entry := range result.Entries {
 		entries = append(entries, FeedEntry{
-			EntryID:   entry.ID,
-			FeedID:    entry.FeedID,
-			YouTubeID: strings.TrimPrefix(entry.URL, "https://www.youtube.com/watch?v="),
+			EntryID:          entry.ID,
+			FeedID:           entry.FeedID,
+			YoutubeChannelID: strings.TrimPrefix(entry.Feed.FeedURL, "https://www.youtube.com/feeds/videos.xml?channel_id="),
+			YoutubeID:        strings.TrimPrefix(entry.URL, "https://www.youtube.com/watch?v="),
 		})
-
-		//	ID:          uuid.New(),
-		//	Status:      model.STATUS_NEW,
-		//	YoutubeURL:  entry.URL,
-		//	FeedID:      strconv.Itoa(int(entry.ID)),
-		//	Title:       entry.Title,
-		//	Description: entry.Content,
-		//})
 	}

 	return entries, nil
--- a/fetcher/summary.go
+++ b/fetcher/summary.go
@ -1,4 +1,4 @@
-package fetcher
+package fetch

 import "ewintr.nl/yogai/model"

--- a/fetch/youtube.go
+++ b/fetch/youtube.go
@ -0,0 +1,76 @@
+package fetch
+
+import (
+	"strings"
+
+	"ewintr.nl/yogai/model"
+	"google.golang.org/api/youtube/v3"
+)
+
+type Youtube struct {
+	Client *youtube.Service
+}
+
+func NewYoutube(client *youtube.Service) *Youtube {
+	return &Youtube{Client: client}
+}
+
+func (y *Youtube) Search(channelID model.YoutubeChannelID, pageToken string) ([]model.YoutubeVideoID, string, error) {
+	call := y.Client.Search.
+		List([]string{"id"}).
+		MaxResults(50).
+		Type("video").
+		Order("date").
+		ChannelId(string(channelID))
+
+	if pageToken != "" {
+		call.PageToken(pageToken)
+	}
+
+	response, err := call.Do()
+	if err != nil {
+		return []model.YoutubeVideoID{}, "", err
+	}
+
+	ids := make([]model.YoutubeVideoID, len(response.Items))
+	for i, item := range response.Items {
+		ids[i] = model.YoutubeVideoID(item.Id.VideoId)
+	}
+
+	return ids, response.NextPageToken, nil
+}
+
+func (y *Youtube) FetchMetadata(ytIDs []model.YoutubeVideoID) (map[model.YoutubeVideoID]Metadata, error) {
+	strIDs := make([]string, len(ytIDs))
+	for i, id := range ytIDs {
+		strIDs[i] = string(id)
+	}
+	call := y.Client.Videos.
+		List([]string{"snippet,contentDetails"}).
+		Id(strings.Join(strIDs, ","))
+
+	response, err := call.Do()
+	if err != nil {
+		return map[model.YoutubeVideoID]Metadata{}, err
+	}
+
+	mds := make(map[model.YoutubeVideoID]Metadata, len(response.Items))
+	for _, item := range response.Items {
+		if item.Snippet == nil {
+			continue
+		}
+		md := Metadata{
+			Title:       item.Snippet.Title,
+			Description: item.Snippet.Description,
+			PublishedAt: item.Snippet.PublishedAt,
+		}
+
+		if item.ContentDetails != nil {
+			md.Duration = item.ContentDetails.Duration
+		}
+
+		mds[model.YoutubeVideoID(item.Id)] = md
+	}
+
+	return mds, nil
+}
--- a/fetcher/feedreader.go
+++ b/fetcher/feedreader.go
@ -1,12 +0,0 @@
-package fetcher
-
-type FeedEntry struct {
-	EntryID   int64
-	FeedID    int64
-	YouTubeID string
-}
-
-type FeedReader interface {
-	Unread() ([]FeedEntry, error)
-	MarkRead(feedID int64) error
-}
--- a/fetcher/fetcher.go
+++ b/fetcher/fetcher.go
@ -1,183 +0,0 @@
-package fetcher
-
-import (
-	"ewintr.nl/yogai/model"
-	"ewintr.nl/yogai/storage"
-	"github.com/google/uuid"
-	"golang.org/x/exp/slog"
-	"time"
-)
-
-type Fetcher struct {
-	interval        time.Duration
-	videoRepo       storage.VideoRepository
-	feedReader      FeedReader
-	metadataFetcher MetadataFetcher
-	summaryFetcher  SummaryFetcher
-	pipeline        chan *model.Video
-	needsMetadata   chan *model.Video
-	needsSummary    chan *model.Video
-	logger          *slog.Logger
-}
-
-func NewFetch(videoRepo storage.VideoRepository, feedReader FeedReader, interval time.Duration, metadataFetcher MetadataFetcher, summaryFetcher SummaryFetcher, logger *slog.Logger) *Fetcher {
-	return &Fetcher{
-		interval:        interval,
-		videoRepo:       videoRepo,
-		feedReader:      feedReader,
-		metadataFetcher: metadataFetcher,
-		summaryFetcher:  summaryFetcher,
-		pipeline:        make(chan *model.Video, 10),
-		needsMetadata:   make(chan *model.Video, 10),
-		needsSummary:    make(chan *model.Video, 10),
-		logger:          logger,
-	}
-}
-
-func (f *Fetcher) Run() {
-	go f.ReadFeeds()
-	go f.MetadataFetcher()
-	go f.SummaryFetcher()
-	go f.FindUnprocessed()
-
-	f.logger.Info("started pipeline")
-	for {
-		select {
-		case video := <-f.pipeline:
-			switch video.Status {
-			case model.StatusNew:
-				f.needsMetadata <- video
-			case model.StatusHasMetadata:
-				f.needsSummary <- video
-			case model.StatusHasSummary:
-				video.Status = model.StatusReady
-				f.logger.Info("video is ready", slog.String("id", video.ID.String()))
-
-			}
-			if err := f.videoRepo.Save(video); err != nil {
-				f.logger.Error("failed to save video", err)
-				continue
-			}
-		}
-
-	}
-}
-
-func (f *Fetcher) FindUnprocessed() {
-	f.logger.Info("looking for unprocessed videos")
-	videos, err := f.videoRepo.FindByStatus(model.StatusNew, model.StatusHasMetadata)
-	if err != nil {
-		f.logger.Error("failed to fetch unprocessed videos", err)
-		return
-	}
-	f.logger.Info("found unprocessed videos", slog.Int("count", len(videos)))
-	for _, video := range videos {
-		f.pipeline <- video
-	}
-}
-
-func (f *Fetcher) ReadFeeds() {
-	f.logger.Info("started feed reader")
-	ticker := time.NewTicker(f.interval)
-	for range ticker.C {
-		entries, err := f.feedReader.Unread()
-		if err != nil {
-			f.logger.Error("failed to fetch unread entries", err)
-			continue
-		}
-		f.logger.Info("fetched unread entries", slog.Int("count", len(entries)))
-		if len(entries) == 0 {
-			continue
-		}
-
-		for _, entry := range entries {
-			video := &model.Video{
-				ID:        uuid.New(),
-				Status:    model.StatusNew,
-				YoutubeID: entry.YouTubeID,
-				// feed id
-			}
-			if err := f.videoRepo.Save(video); err != nil {
-				f.logger.Error("failed to save video", err)
-				continue
-			}
-			f.pipeline <- video
-			if err := f.feedReader.MarkRead(entry.EntryID); err != nil {
-				f.logger.Error("failed to mark entry as read", err)
-				continue
-			}
-		}
-	}
-}
-
-func (f *Fetcher) MetadataFetcher() {
-	f.logger.Info("started metadata fetcher")
-
-	buffer := []*model.Video{}
-	timeout := time.NewTimer(10 * time.Second)
-	fetch := make(chan []*model.Video)
-
-	go func() {
-		for videos := range fetch {
-			f.logger.Info("fetching metadata", slog.Int("count", len(videos)))
-			ids := make([]string, 0, len(videos))
-			for _, video := range videos {
-				ids = append(ids, video.YoutubeID)
-			}
-			mds, err := f.metadataFetcher.FetchMetadata(ids)
-			if err != nil {
-				f.logger.Error("failed to fetch metadata", err)
-				continue
-			}
-			for _, video := range videos {
-				video.Title = mds[video.YoutubeID].Title
-				video.Description = mds[video.YoutubeID].Description
-				video.Status = model.StatusHasMetadata
-
-				if err := f.videoRepo.Save(video); err != nil {
-					f.logger.Error("failed to save video", err)
-					continue
-				}
-			}
-			f.logger.Info("fetched metadata", slog.Int("count", len(videos)))
-		}
-	}()
-
-	for {
-		select {
-		case video := <-f.needsMetadata:
-			timeout.Reset(10 * time.Second)
-			buffer = append(buffer, video)
-			if len(buffer) >= 10 {
-				batch := make([]*model.Video, len(buffer))
-				copy(batch, buffer)
-				fetch <- batch
-				buffer = []*model.Video{}
-			}
-		case <-timeout.C:
-			if len(buffer) == 0 {
-				continue
-			}
-			batch := make([]*model.Video, len(buffer))
-			copy(batch, buffer)
-			fetch <- batch
-			buffer = []*model.Video{}
-		}
-	}
-}
-
-func (f *Fetcher) SummaryFetcher() {
-	for {
-		select {
-		case video := <-f.needsSummary:
-			f.logger.Info("fetching summary", slog.String("id", video.ID.String()))
-			if err := f.summaryFetcher.FetchSummary(video); err != nil {
-				f.logger.Error("failed to fetch summary", err)
-				continue
-			}
-			video.Status = model.StatusHasSummary
-			f.logger.Info("fetched summary", slog.String("id", video.ID.String()))
-			f.pipeline <- video
-		}
-	}
-}
--- a/fetcher/metadata.go
+++ b/fetcher/metadata.go
@ -1,10 +0,0 @@
-package fetcher
-
-type Metadata struct {
-	Title       string
-	Description string
-}
-
-type MetadataFetcher interface {
-	FetchMetadata([]string) (map[string]Metadata, error)
-}
--- a/fetcher/youtube.go
+++ b/fetcher/youtube.go
@ -1,35 +0,0 @@
-package fetcher
-
-import (
-	"google.golang.org/api/youtube/v3"
-	"strings"
-)
-
-type Youtube struct {
-	Client *youtube.Service
-}
-
-func NewYoutube(client *youtube.Service) *Youtube {
-	return &Youtube{Client: client}
-}
-
-func (y *Youtube) FetchMetadata(ytIDs []string) (map[string]Metadata, error) {
-	call := y.Client.Videos.
-		List([]string{"snippet"}).
-		Id(strings.Join(ytIDs, ","))
-
-	response, err := call.Do()
-	if err != nil {
-		return map[string]Metadata{}, err
-	}
-
-	mds := make(map[string]Metadata, len(response.Items))
-	for _, item := range response.Items {
-		mds[item.Id] = Metadata{
-			Title:       item.Snippet.Title,
-			Description: item.Snippet.Description,
-		}
-	}
-
-	return mds, nil
-}
--- a/go.mod
+++ b/go.mod
@ -6,6 +6,8 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/lib/pq v1.10.9
 	github.com/sashabaranov/go-openai v1.9.4
+	github.com/weaviate/weaviate v1.19.0
+	github.com/weaviate/weaviate-go-client/v4 v4.8.1
 	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
 	google.golang.org/api v0.122.0
 	miniflux.app v0.0.0-20230505000442-88062ab9f959
@ -14,11 +16,28 @@ require (
 require (
 	cloud.google.com/go/compute v1.19.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
+	github.com/go-openapi/analysis v0.21.2 // indirect
+	github.com/go-openapi/errors v0.20.3 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.6 // indirect
+	github.com/go-openapi/loads v0.21.1 // indirect
+	github.com/go-openapi/spec v0.20.4 // indirect
+	github.com/go-openapi/strfmt v0.21.3 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/validate v0.21.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/s2a-go v0.1.3 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/googleapis/gax-go/v2 v2.8.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
+	go.mongodb.org/mongo-driver v1.11.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/crypto v0.8.0 // indirect
 	golang.org/x/net v0.9.0 // indirect
@ -29,4 +48,5 @@ require (
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
 	google.golang.org/grpc v1.54.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
--- a/go.sum
+++ b/go.sum
@ -7,7 +7,14 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
+github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ=
+github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@ -17,7 +24,9 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@ -26,9 +35,62 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU=
+github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY=
+github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc=
+github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=
+github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
+github.com/go-openapi/loads v0.21.1 h1:Wb3nVZpdEzDTcly8S4HMkey6fjARRzb7iEaySimlDW0=
+github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g=
+github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=
+github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
+github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg=
+github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k=
+github.com/go-openapi/strfmt v0.21.3 h1:xwhj5X6CjXEZZHMWy1zKJxvW9AfHC9pkyUjLvHtKG7o=
+github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/validate v0.21.0 h1:+Wqk39yKOhfpLqNLEC0/eViCkzM5FVXVqrvt526+wcI=
+github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
+github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
+github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
+github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
+github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
+github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
+github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
+github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
+github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
+github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
+github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
+github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
+github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
+github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
+github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -46,16 +108,19 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/s2a-go v0.1.3 h1:FAgZmpLl/SXurPEZyCMPBIiiYeTbqfjlbdnCNTAkbGE=
 github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -64,29 +129,98 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5
 github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc=
 github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sashabaranov/go-openai v1.9.4 h1:KanoCEoowAI45jVXlenMCckutSRr39qOmSi9MyPBfZM=
 github.com/sashabaranov/go-openai v1.9.4/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/weaviate/weaviate v1.19.0 h1:JKmScZZ5VWVESCkji37bT1cNFCCRIZrne7ENoRHT1vM=
+github.com/weaviate/weaviate v1.19.0/go.mod h1:hvgLEEiZx0gQNEDLNgPGssk8UQjc/CDxZv2Dd5SYgs8=
+github.com/weaviate/weaviate-go-client/v4 v4.8.1 h1:oYU+tS9cRyjB0OLV55oN7pnu+EGfa+yIndo3SVlpWJs=
+github.com/weaviate/weaviate-go-client/v4 v4.8.1/go.mod h1:5vfV3ZVIpG48S18vFi85RR+BFq/YvwN9RCUYoU3oVL0=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
+go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
+go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8=
+go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y=
+go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
 golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -107,6 +241,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
@ -118,15 +253,25 @@ golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -138,6 +283,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
@ -147,7 +293,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -189,9 +339,20 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
--- a/handler/handler.go
+++ b/handler/handler.go
@ -1 +1,48 @@
 package handler
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+func Index(w http.ResponseWriter) {
+	Message(w, http.StatusOK, "yogai index")
+}
+
+func Message(w http.ResponseWriter, status int, message string, details ...any) {
+	w.WriteHeader(status)
+	response := struct {
+		Message string `json:"message"`
+		Details []any  `json:"details,omitempty"`
+	}{
+		Message: message,
+		Details: details,
+	}
+	body, marshalErr := json.Marshal(response)
+	if marshalErr != nil {
+		fmt.Fprintf(w, fmt.Sprintf(`{"message": %q, "details":%q}`, message, marshalErr.Error()))
+		return
+	}
+	fmt.Fprintf(w, string(body))
+}
+
+func Error(w http.ResponseWriter, status int, message string, err error, details ...any) {
+	w.WriteHeader(status)
+	response := struct {
+		Message string `json:"message"`
+		Error   string `json:"error"`
+		Details []any  `json:"details,omitempty"`
+	}{
+		Message: message,
+		Error:   err.Error(),
+		Details: details,
+	}
+	body, marshalErr := json.Marshal(response)
+	if marshalErr != nil {
+		fmt.Fprintf(w, fmt.Sprintf(`{"message": %q, "error": %q, "details":%q}`, message, err.Error(), marshalErr.Error()))
+		return
+	}
+
+	fmt.Fprintf(w, string(body))
+}
--- a/handler/server.go
+++ b/handler/server.go
@ -0,0 +1,82 @@
+package handler
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"path"
+	"strings"
+
+	"ewintr.nl/yogai/storage"
+	"golang.org/x/exp/slog"
+	"miniflux.app/logger"
+)
+
+type Server struct {
+	apis   map[string]http.Handler
+	logger *slog.Logger
+}
+
+func NewServer(videoRepo storage.VideoRelRepository, logger *slog.Logger) *Server {
+	return &Server{
+		apis: map[string]http.Handler{
+			"video": NewVideoAPI(videoRepo, logger),
+		},
+		logger: logger,
+	}
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	originalPath := r.URL.Path
+	rec := httptest.NewRecorder() // records the response to be able to mix writing headers and content
+
+	w.Header().Add("Content-Type", "application/json")
+
+	// route to api
+	head, tail := ShiftPath(r.URL.Path)
+	if len(head) == 0 {
+		Index(rec)
+		returnResponse(w, rec)
+		return
+	}
+	api, ok := s.apis[head]
+	if !ok {
+		Error(rec, http.StatusNotFound, "Not found", fmt.Errorf("%s is not a valid path", r.URL.Path))
+	} else {
+		r.URL.Path = tail
+		api.ServeHTTP(rec, r)
+	}
+
+	returnResponse(w, rec)
+	logger.Info("request served", "path", originalPath, "status", rec.Code)
+}
+
+func returnResponse(w http.ResponseWriter, rec *httptest.ResponseRecorder) {
+	w.WriteHeader(rec.Code)
+	for k, v := range rec.Header() {
+		w.Header()[k] = v
+	}
+	w.Write(rec.Body.Bytes())
+}
+
+// ShiftPath splits off the first component of p, which will be cleaned of
+// relative components before processing. head will never contain a slash and
+// tail will always be a rooted path without trailing slash.
+// See https://blog.merovius.de/posts/2017-06-18-how-not-to-use-an-http-router/
+func ShiftPath(p string) (string, string) {
+	p = path.Clean("/" + p)
+
+	// restore iri prefixes that might be mangled by path.Clean
+	for k, v := range map[string]string{
+		"http:/":  "http://",
+		"https:/": "https://",
+	} {
+		p = strings.Replace(p, k, v, -1)
+	}
+
+	i := strings.Index(p[1:], "/") + 1
+	if i <= 0 {
+		return p[1:], "/"
+	}
+	return p[1:i], p[i:]
+}
--- a/handler/server.go].go
+++ b/handler/server.go].go
@ -1 +0,0 @@
-package handler
--- a/handler/video.go
+++ b/handler/video.go
@ -0,0 +1,71 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"ewintr.nl/yogai/model"
+	"ewintr.nl/yogai/storage"
+	"golang.org/x/exp/slog"
+)
+
+type VideoAPI struct {
+	videoRepo storage.VideoRelRepository
+	logger    *slog.Logger
+}
+
+func NewVideoAPI(videoRepo storage.VideoRelRepository, logger *slog.Logger) *VideoAPI {
+	return &VideoAPI{
+		videoRepo: videoRepo,
+		logger:    logger,
+	}
+}
+
+func (v *VideoAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	videoID, _ := ShiftPath(r.URL.Path)
+
+	switch {
+	case r.Method == http.MethodGet && videoID == "":
+		v.List(w, r)
+	default:
+		Error(w, http.StatusNotFound, "not found", fmt.Errorf("method %s with subpath %q was not registered in the repository api", r.Method, videoID))
+	}
+}
+
+func (v *VideoAPI) List(w http.ResponseWriter, r *http.Request) {
+	video, err := v.videoRepo.FindByStatus(model.StatusReady)
+	if err != nil {
+		v.returnErr(r.Context(), w, http.StatusInternalServerError, "could not list repositories", err)
+		return
+	}
+
+	type respVideo struct {
+		YoutubeID string `json:"youtube_url"`
+		Title     string `json:"title"`
+		Summary   string `json:"summary"`
+	}
+	var resp []respVideo
+	for _, v := range video {
+		resp = append(resp, respVideo{
+			YoutubeID: string(v.YoutubeID),
+			Title:     v.YoutubeTitle,
+			Summary:   v.Summary,
+		})
+	}
+
+	jsonBody, err := json.Marshal(resp)
+	if err != nil {
+		v.returnErr(r.Context(), w, http.StatusInternalServerError, "could not marshal response", err)
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprintf(w, string(jsonBody))
+}
+
+func (v *VideoAPI) returnErr(_ context.Context, w http.ResponseWriter, status int, message string, err error, details ...any) {
+	v.logger.Error(message, slog.String("err", err.Error()), slog.String("details", fmt.Sprintf("%+v", details)))
+	Error(w, status, message, err, details...)
+}
--- a/model/feed.go
+++ b/model/feed.go
@ -0,0 +1,17 @@
+package model
+
+import "github.com/google/uuid"
+
+type FeedStatus string
+
+const (
+	FeedStatusNew   FeedStatus = "new"
+	FeedStatusReady FeedStatus = "ready"
+)
+
+type Feed struct {
+	ID               uuid.UUID
+	Status           FeedStatus
+	Title            string
+	YoutubeChannelID YoutubeChannelID
+}
--- a/model/video.go
+++ b/model/video.go
@ -2,21 +2,32 @@ package model

 import "github.com/google/uuid"

-type Status string
+type VideoStatus string

 const (
-	StatusNew         Status = "new"
-	StatusHasMetadata Status = "has_metadata"
-	StatusHasSummary  Status = "has_summary"
-	StatusReady       Status = "ready"
+	StatusNew     VideoStatus = "new"
+	StatusFetched VideoStatus = "fetched"
+	StatusReady   VideoStatus = "ready"
 )

+type YoutubeVideoID string
+
+type YoutubeChannelID string
+
 type Video struct {
-	ID          uuid.UUID
-	Status      Status
-	YoutubeID   string
-	FeedID      uuid.UUID
-	Title       string
-	Description string
-	Summary     string
+	ID                 uuid.UUID
+	Status             VideoStatus
+	YoutubeID          YoutubeVideoID
+	YoutubeChannelID   YoutubeChannelID
+	YoutubeTitle       string
+	YoutubeDescription string
+	YoutubeDuration    string
+	YoutubePublishedAt string
+
+	Summary string
+}
+
+type VideoVec struct {
+	ID      uuid.UUID
+	Summary string
 }
--- a/process/openai.go
+++ b/process/openai.go
@ -1,29 +1,34 @@
-package fetcher
+package process

 import (
 	"context"
-	"ewintr.nl/yogai/model"
 	"fmt"
+
+	"ewintr.nl/yogai/model"
 	"github.com/sashabaranov/go-openai"
 )

-const summarizePrompt = `You are an helpful assistant. Your task is to extract all text that refers to the content of a yoga workout video from the description a user gives you.
-You will not add introductory sentences like "This text is about", or "Summary of...". Just give the words verbatim. Trim any white space back to a simple space
-`
-
-type OpenAI struct {
+type OpenAISummarizer struct {
 	client *openai.Client
 }

-func NewOpenAI(apiKey string) *OpenAI {
-	return &OpenAI{
-		client: openai.NewClient(apiKey),
+func NewOpenAISummarizer(client *openai.Client) *OpenAISummarizer {
+	return &OpenAISummarizer{
+		client: client,
 	}
 }

-func (o *OpenAI) FetchSummary(video *model.Video) error {
-	resp, err := o.client.CreateChatCompletion(
-		context.Background(),
+func (sum *OpenAISummarizer) Name() string {
+	return "openai summarizer"
+}
+
+func (sum *OpenAISummarizer) Do(ctx context.Context, video *model.Video) error {
+	const summarizePrompt = `You are an helpful assistant. Your task is to extract all text that refers to the content of a yoga workout video from the description a user gives you.
+You will not add introductory sentences like "This text is about", or "Summary of...". Just give the words verbatim. Trim any white space back to a simple space
+`
+
+	resp, err := sum.client.CreateChatCompletion(
+		ctx,
 		openai.ChatCompletionRequest{
 			Model: openai.GPT4,
 			Messages: []openai.ChatCompletionMessage{
@ -34,7 +39,7 @@ func (o *OpenAI) FetchSummary(video *model.Video) error {

 				{
 					Role:    openai.ChatMessageRoleUser,
-					Content: fmt.Sprintf("%s\n\n%s", video.Title, video.Description),
+					Content: fmt.Sprintf("%s\n\n%s", video.YoutubeTitle, video.YoutubeDescription),
 				},
 			},
 		})
--- a/process/processor.go
+++ b/process/processor.go
@ -0,0 +1,89 @@
+package process
+
+import (
+	"context"
+
+	"ewintr.nl/yogai/model"
+	"ewintr.nl/yogai/storage"
+	"github.com/sashabaranov/go-openai"
+	"golang.org/x/exp/slog"
+)
+
+type VideoProcessor interface {
+	Name() string
+	Do(ctx context.Context, video *model.Video) error
+}
+
+type Processors struct {
+	procs map[string]VideoProcessor
+}
+
+func NewProcessors(openAIClient *openai.Client) *Processors {
+	return &Processors{
+		procs: map[string]VideoProcessor{
+			"summarizer": NewOpenAISummarizer(openAIClient),
+		},
+	}
+}
+
+func (p *Processors) Next(video *model.Video) VideoProcessor {
+	if video.Summary == "" {
+		return p.procs["summarizer"]
+	}
+
+	return nil
+}
+
+type Pipeline struct {
+	in         chan *model.Video
+	procs      *Processors
+	logger     *slog.Logger
+	relStorage storage.VideoRelRepository
+	vecStorage storage.VideoVecRepository
+}
+
+func NewPipeline(in chan *model.Video, processors *Processors, relDB storage.VideoRelRepository, vecDB storage.VideoVecRepository, logger *slog.Logger) *Pipeline {
+	return &Pipeline{
+		in:         in,
+		procs:      processors,
+		relStorage: relDB,
+		vecStorage: vecDB,
+		logger:     logger,
+	}
+}
+
+func (p *Pipeline) Run() {
+	ctx := context.Background()
+	for video := range p.in {
+		p.Process(ctx, video)
+	}
+}
+
+func (p *Pipeline) Process(ctx context.Context, video *model.Video) {
+	p.logger.Info("processing video", slog.String("video", string(video.YoutubeID)))
+	for {
+		next := p.procs.Next(video)
+		if next == nil {
+			p.logger.Info("no more processors for video", slog.String("video", string(video.YoutubeID)))
+			video.Status = model.StatusReady
+			if err := p.relStorage.Save(video); err != nil {
+				p.logger.Error("failed to save video in rel db", slog.String("video", string(video.YoutubeID)), slog.String("error", err.Error()))
+			}
+			return
+		}
+
+		p.logger.Info("processing video", slog.String("video", string(video.YoutubeID)), slog.String("processor", next.Name()))
+		if err := next.Do(context.Background(), video); err != nil {
+			p.logger.Error("failed to process video", slog.String("video", string(video.YoutubeID)), slog.String("processor", next.Name()), slog.String("error", err.Error()))
+			return
+		}
+		if err := p.relStorage.Save(video); err != nil {
+			p.logger.Error("failed to save video in rel db", slog.String("video", string(video.YoutubeID)), slog.String("error", err.Error()))
+			return
+		}
+		if err := p.vecStorage.Save(ctx, video); err != nil {
+			p.logger.Error("failed to save video in rel db", slog.String("video", string(video.YoutubeID)), slog.String("error", err.Error()))
+			return
+		}
+	}
+}
--- a/service.go
+++ b/service.go
@ -2,19 +2,28 @@ package main

 import (
 	"context"
-	"ewintr.nl/yogai/fetcher"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"time"
+
+	"ewintr.nl/yogai/fetch"
+	"ewintr.nl/yogai/handler"
+	"ewintr.nl/yogai/process"
 	"ewintr.nl/yogai/storage"
+	"github.com/sashabaranov/go-openai"
 	"golang.org/x/exp/slog"
 	"google.golang.org/api/option"
 	"google.golang.org/api/youtube/v3"
-	"os"
-	"os/signal"
-	"time"
 )

 func main() {
+
 	ctx := context.Background()
-	logger := slog.New(slog.NewTextHandler(os.Stderr))
+	logger := slog.New(slog.NewTextHandler(os.Stdout))
+
 	postgres, err := storage.NewPostgres(storage.PostgresInfo{
 		Host:     getParam("POSTGRES_HOST", "localhost"),
 		Port:     getParam("POSTGRES_PORT", "5432"),
@ -26,9 +35,10 @@ func main() {
 		logger.Error("unable to connect to postgres", err)
 		os.Exit(1)
 	}
-	videoRepo := storage.NewPostgresVideoRepository(postgres)
+	videoRelRepo := storage.NewPostgresVideoRepository(postgres)
+	feedRelRepo := storage.NewPostgresFeedRepository(postgres)

-	mflx := fetcher.NewMiniflux(fetcher.MinifluxInfo{
+	mflxClient := fetch.NewMiniflux(fetch.MinifluxInfo{
 		Endpoint: getParam("MINIFLUX_ENDPOINT", "http://localhost/v1"),
 		ApiKey:   getParam("MINIFLUX_APIKEY", ""),
 	})
@ -39,18 +49,47 @@ func main() {
 		os.Exit(1)
 	}

-	ytClient, err := youtube.NewService(ctx, option.WithAPIKey(getParam("YOUTUBE_API_KEY", "")))
+	yt, err := youtube.NewService(ctx, option.WithAPIKey(getParam("YOUTUBE_API_KEY", "")))
 	if err != nil {
 		logger.Error("unable to create youtube service", err)
 		os.Exit(1)
 	}
-	yt := fetcher.NewYoutube(ytClient)
+	ytClient := fetch.NewYoutube(yt)

-	openAIClient := fetcher.NewOpenAI(getParam("OPENAI_API_KEY", ""))
+	openaiKey := getParam("OPENAI_API_KEY", "")
+	openAIClient := openai.NewClient(openaiKey)

-	fetcher := fetcher.NewFetch(videoRepo, mflx, fetchInterval, yt, openAIClient, logger)
+	wvResetSchema := getParam("WEAVIATE_RESET_SCHEMA", "false") == "true"
+	wvClient, err := storage.NewWeaviate(getParam("WEAVIATE_HOST", ""), getParam("WEAVIATE_API_KEY", ""), openaiKey)
+	if err != nil {
+		logger.Error("unable to create weaviate client", err)
+		os.Exit(1)
+	}
+	if wvResetSchema {
+		logger.Info("resetting weaviate schema")
+		if err := wvClient.ResetSchema(); err != nil {
+			logger.Error("unable to reset weaviate schema", err)
+			os.Exit(1)
+		}
+	}
+
+	fetcher := fetch.NewFetch(feedRelRepo, videoRelRepo, ytClient, mflxClient, fetchInterval, ytClient, logger)
 	go fetcher.Run()
-	logger.Info("service started")
+	logger.Info("fetch service started")
+
+	procs := process.NewProcessors(openAIClient)
+	for i := 0; i < 4; i++ {
+		go process.NewPipeline(fetcher.Out(), procs, videoRelRepo, wvClient, logger.With(slog.Int("pipeline", i))).Run()
+	}
+	logger.Info("processing service started")
+
+	port, err := strconv.Atoi(getParam("API_PORT", "8080"))
+	if err != nil {
+		logger.Error("invalid port", err)
+		os.Exit(1)
+	}
+	go http.ListenAndServe(fmt.Sprintf(":%d", port), handler.NewServer(videoRelRepo, logger))
+	logger.Info("http server started")

 	done := make(chan os.Signal)
 	signal.Notify(done, os.Interrupt)
--- a/storage/migrations.go
+++ b/storage/migrations.go
@ -0,0 +1,55 @@
+package storage
+
+var pgMigration = []string{
+	`CREATE TYPE video_status AS ENUM ('new', 'ready')`,
+	`CREATE TABLE video (
+id uuid PRIMARY KEY,
+status video_status NOT NULL,
+youtube_id VARCHAR(255) NOT NULL UNIQUE,
+title VARCHAR(255) NOT NULL,
+feed_id VARCHAR(255) NOT NULL,
+description TEXT,
+summary TEXT
+)`,
+	`CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`,
+	`ALTER TABLE video
+ALTER COLUMN status TYPE video_status_new
+USING video::text::video_status_new`,
+	`DROP TYPE video_status`,
+	`ALTER TYPE video_status_new RENAME TO video_status`,
+	`UPDATE video SET summary = '' WHERE summary IS NULL `,
+	`UPDATE video SET description = '' WHERE description IS NULL `,
+	`ALTER TABLE video 
+ALTER COLUMN summary SET DEFAULT '', 
+ALTER COLUMN summary SET NOT NULL,
+ALTER COLUMN description SET DEFAULT '', 
+ALTER COLUMN description SET NOT NULL`,
+	`CREATE TYPE feed_status AS ENUM ('new', 'ready')`,
+	`CREATE TABLE feed (
+id uuid PRIMARY KEY,
+status feed_status NOT NULL,
+youtube_channel_id VARCHAR(255) NOT NULL UNIQUE,
+title VARCHAR(255) NOT NULL
+)`,
+	`ALTER TABLE video
+DROP COLUMN feed_id,
+ADD COLUMN youtube_channel_id VARCHAR(255) NOT NULL REFERENCES feed(youtube_channel_id)`,
+	`ALTER TABLE video
+ADD COLUMN duration VARCHAR(255),
+ADD COLUMN published_at VARCHAR(255)`,
+	`ALTER TABLE video RENAME COLUMN duration TO youtube_duration`,
+	`ALTER TABLE video RENAME COLUMN published_at TO youtube_published_id`,
+	`ALTER TABLE video RENAME COLUMN title TO youtube_title`,
+	`ALTER TABLE video RENAME COLUMN description TO youtube_description`,
+	`ALTER TABLE video RENAME COLUMN youtube_published_id TO youtube_published_at`,
+	`UPDATE video SET status = 'new'`,
+	`CREATE TYPE video_status_new AS ENUM ('new', 'fetched', 'ready')`,
+	`BEGIN;
+ALTER TABLE video ADD COLUMN status_new video_status_new;
+UPDATE video SET status_new = status::text::video_status_new;
+ALTER TABLE video DROP COLUMN status;
+ALTER TABLE video RENAME COLUMN status_new TO status;
+COMMIT;`,
+	`DROP TYPE video_status`,
+	`ALTER TYPE video_status_new RENAME TO video_status`,
+}
--- a/storage/postgres.go
+++ b/storage/postgres.go
@ -2,8 +2,9 @@ package storage

 import (
 	"database/sql"
-	"ewintr.nl/yogai/model"
 	"fmt"
+
+	"ewintr.nl/yogai/model"
 	"github.com/lib/pq"
 	_ "github.com/lib/pq"
 )
@ -43,24 +44,26 @@ func NewPostgresVideoRepository(postgres *Postgres) *PostgresVideoRepository {
 }

 func (p *PostgresVideoRepository) Save(v *model.Video) error {
-	query := `INSERT INTO video (id, status, youtube_id, feed_id, title, description, summary)
-VALUES ($1, $2, $3, $4, $5, $6, $7)
+	query := `INSERT INTO video (id, status, youtube_id, youtube_channel_id, youtube_title, youtube_description, youtube_duration, youtube_published_at, summary)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
 ON CONFLICT (id)
 DO UPDATE SET
  id = EXCLUDED.id,
  status = EXCLUDED.status,
  youtube_id = EXCLUDED.youtube_id,
-  feed_id = EXCLUDED.feed_id,
-  title = EXCLUDED.title,
-  description = EXCLUDED.description,
+  youtube_channel_id = EXCLUDED.youtube_channel_id,
+  youtube_title = EXCLUDED.youtube_title,
+  youtube_description = EXCLUDED.youtube_description,
+  youtube_duration = EXCLUDED.youtube_duration,
+  youtube_published_at = EXCLUDED.youtube_published_at,
  summary = EXCLUDED.summary;`
-	_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.FeedID, v.Title, v.Description, v.Summary)
+	_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.YoutubeChannelID, v.YoutubeTitle, v.YoutubeDescription, v.YoutubeDuration, v.YoutubePublishedAt, v.Summary)

 	return err
 }

-func (p *PostgresVideoRepository) FindByStatus(statuses ...model.Status) ([]*model.Video, error) {
-	query := `SELECT id, status, youtube_id, feed_id, title, description, summary
+func (p *PostgresVideoRepository) FindByStatus(statuses ...model.VideoStatus) ([]*model.Video, error) {
+	query := `SELECT id, status, youtube_channel_id, youtube_id, youtube_title, youtube_description,youtube_duration, youtube_published_at, summary
 FROM video
 WHERE status = ANY($1)`
 	rows, err := p.db.Query(query, pq.Array(statuses))
@ -71,7 +74,7 @@ WHERE status = ANY($1)`
 	videos := []*model.Video{}
 	for rows.Next() {
 		v := &model.Video{}
-		if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeID, &v.FeedID, &v.Title, &v.Description, &v.Summary); err != nil {
+		if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeChannelID, &v.YoutubeID, &v.YoutubeTitle, &v.YoutubeDescription, &v.YoutubeDuration, &v.YoutubePublishedAt, &v.Summary); err != nil {
 			return nil, err
 		}
 		videos = append(videos, v)
@ -81,30 +84,48 @@ WHERE status = ANY($1)`
 	return videos, nil
 }

-var pgMigration = []string{
-	`CREATE TYPE video_status AS ENUM ('new', 'ready')`,
-	`CREATE TABLE video (
-id uuid PRIMARY KEY,
-status video_status NOT NULL,
-youtube_id VARCHAR(255) NOT NULL UNIQUE,
-title VARCHAR(255) NOT NULL,
-feed_id VARCHAR(255) NOT NULL,
-description TEXT,
-summary TEXT
-)`,
-	`CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`,
-	`ALTER TABLE video
-ALTER COLUMN status TYPE video_status_new
-USING video::text::video_status_new`,
-	`DROP TYPE video_status`,
-	`ALTER TYPE video_status_new RENAME TO video_status`,
-	`UPDATE video SET summary = '' WHERE summary IS NULL `,
-	`UPDATE video SET description = '' WHERE description IS NULL `,
-	`ALTER TABLE video 
-ALTER COLUMN summary SET DEFAULT '', 
-ALTER COLUMN summary SET NOT NULL,
-ALTER COLUMN description SET DEFAULT '', 
-ALTER COLUMN description SET NOT NULL`,
+type PostgresFeedRepository struct {
+	*Postgres
+}
+
+func NewPostgresFeedRepository(postgres *Postgres) *PostgresFeedRepository {
+	return &PostgresFeedRepository{postgres}
+}
+
+func (p *PostgresFeedRepository) Save(f *model.Feed) error {
+	query := `INSERT INTO feed (id, status, youtube_channel_id, title)
+VALUES ($1, $2, $3, $4)
+ON CONFLICT (id)
+DO UPDATE SET
+  id = EXCLUDED.id,
+  status = EXCLUDED.status,
+  youtube_channel_id = EXCLUDED.youtube_channel_id,
+  title = EXCLUDED.title;`
+	_, err := p.db.Exec(query, f.ID, f.Status, f.YoutubeChannelID, f.Title)
+
+	return err
+}
+
+func (p *PostgresFeedRepository) FindByStatus(statuses ...model.FeedStatus) ([]*model.Feed, error) {
+	query := `SELECT id, status, youtube_channel_id, title
+FROM feed
+WHERE status = ANY($1)`
+	rows, err := p.db.Query(query, pq.Array(statuses))
+	if err != nil {
+		return nil, err
+	}
+
+	feeds := []*model.Feed{}
+	for rows.Next() {
+		f := &model.Feed{}
+		if err := rows.Scan(&f.ID, &f.Status, &f.YoutubeChannelID, &f.Title); err != nil {
+			return nil, err
+		}
+		feeds = append(feeds, f)
+	}
+	rows.Close()
+
+	return feeds, nil
 }

 func (p *Postgres) migrate(wanted []string) error {
--- a/storage/storage.go
+++ b/storage/storage.go
@ -1,10 +1,21 @@
 package storage

 import (
+	"context"
+
 	"ewintr.nl/yogai/model"
 )

-type VideoRepository interface {
-	Save(video *model.Video) error
-	FindByStatus(statuses ...model.Status) ([]*model.Video, error)
+type FeedRelRepository interface {
+	Save(feed *model.Feed) error
+	FindByStatus(statuses ...model.FeedStatus) ([]*model.Feed, error)
+}
+
+type VideoRelRepository interface {
+	Save(video *model.Video) error
+	FindByStatus(statuses ...model.VideoStatus) ([]*model.Video, error)
+}
+
+type VideoVecRepository interface {
+	Save(ctx context.Context, video *model.Video) error
 }
--- a/storage/weaviate.go
+++ b/storage/weaviate.go
@ -0,0 +1,99 @@
+package storage
+
+import (
+	"context"
+	"net/http"
+
+	"ewintr.nl/yogai/model"
+	"github.com/weaviate/weaviate-go-client/v4/weaviate"
+	"github.com/weaviate/weaviate-go-client/v4/weaviate/auth"
+	"github.com/weaviate/weaviate-go-client/v4/weaviate/fault"
+	"github.com/weaviate/weaviate/entities/models"
+)
+
+const (
+	className = "Video"
+)
+
+type Weaviate struct {
+	client *weaviate.Client
+}
+
+func NewWeaviate(host, weaviateApiKey, openaiApiKey string) (*Weaviate, error) {
+	config := weaviate.Config{
+		Scheme:     "https",
+		Host:       host,
+		AuthConfig: auth.ApiKey{Value: weaviateApiKey},
+		Headers: map[string]string{
+			"X-OpenAI-Api-Key": openaiApiKey,
+		},
+	}
+
+	c, err := weaviate.NewClient(config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Weaviate{client: c}, nil
+}
+
+func (w *Weaviate) ResetSchema() error {
+
+	// delete old
+	if err := w.client.Schema().ClassDeleter().WithClassName(className).Do(context.Background()); err != nil {
+		// Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
+		if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
+			return err
+		}
+	}
+
+	// create new
+	classObj := &models.Class{
+		Class:      className,
+		Vectorizer: "text2vec-openai",
+		ModuleConfig: map[string]any{
+			"text2vec-openai": map[string]any{
+				"model":        "ada",
+				"modelVersion": "002",
+				"type":         "text",
+			},
+		},
+	}
+
+	return w.client.Schema().ClassCreator().WithClass(classObj).Do(context.Background())
+}
+
+func (w *Weaviate) Save(ctx context.Context, video *model.Video) error {
+	vec := model.VideoVec{
+		ID:      video.ID,
+		Summary: video.Summary,
+	}
+	vID := vec.ID.String()
+	// check it already exists
+	exists, err := w.client.Data().
+		Checker().
+		WithID(vID).
+		WithClassName(className).
+		Do(ctx)
+	if err != nil {
+		return err
+	}
+
+	if exists {
+		return w.client.Data().
+			Updater().
+			WithID(vID).
+			WithClassName(className).
+			WithProperties(vec).
+			Do(ctx)
+	}
+
+	_, err = w.client.Data().
+		Creator().
+		WithClassName(className).
+		WithID(vID).
+		WithProperties(vec).
+		Do(ctx)
+
+	return err
+}
Author	SHA1	Message	Date
Erik Winter	9eddbac6f9	fix fetch to process	2023-07-06 14:30:37 +02:00
Erik Winter	937eb32e93	vec model and fix log	2023-07-06 13:58:12 +02:00
Erik Winter	f4c856df68	multiple pipelines	2023-07-06 13:37:55 +02:00
Erik Winter	5d125c0ac1	split fetch and process	2023-07-06 13:25:51 +02:00
Erik Winter	3c451ae0a1	store in weaviate	2023-07-04 19:56:44 +02:00
Erik Winter	82d95e98c9	more youtube metadata	2023-05-31 16:27:35 +02:00
Erik Winter	b8c10a9d58	docker file	2023-05-31 15:46:58 +02:00
Erik Winter	b892066abf	fetch historical videos from channel	2023-05-27 14:36:22 +02:00
Erik Winter	f58afaf70b	repo sync test	2023-05-25 12:27:47 +02:00
Erik Winter	10bd39d915	handler index	2023-05-13 13:03:48 +02:00