Dockerfile

@@ -1,11 +0,0 @@
-FROM golang:1.20-alpine as build
-
-WORKDIR /src
-COPY . ./
-RUN go mod download
-RUN go build -o /yogai ./service.go
-
-FROM golang:1.20-alpine
-
-COPY --from=build /yogai /yogai
-CMD /yogai

Makefile

@@ -1,4 +0,0 @@
-docker-push:
-	docker build . -t yogai
-	docker tag yogai registry.ewintr.nl/yogai
-	docker push registry.ewintr.nl/yogai

fetch/feedreader.go

@@ -1,19 +0,0 @@
-package fetch
-
-import "ewintr.nl/yogai/model"
-
-type FeedEntry struct {
-	EntryID          int64
-	FeedID           int64
-	YoutubeChannelID string
-	YoutubeID        string
-}
-
-type ChannelReader interface {
-	Search(channelID model.YoutubeChannelID, pageToken string) ([]model.YoutubeVideoID, string, error)
-}
-
-type FeedReader interface {
-	Unread() ([]FeedEntry, error)
-	MarkRead(feedID int64) error
-}

fetch/fetcher.go

@@ -1,235 +0,0 @@
-package fetch
-
-import (
-	"time"
-
-	"ewintr.nl/yogai/model"
-	"ewintr.nl/yogai/storage"
-	"github.com/google/uuid"
-	"golang.org/x/exp/slog"
-)
-
-type Fetcher struct {
-	interval        time.Duration
-	feedRepo        storage.FeedRelRepository
-	videoRepo       storage.VideoRelRepository
-	feedReader      FeedReader
-	channelReader   ChannelReader
-	metadataFetcher MetadataFetcher
-	feedPipeline    chan *model.Feed
-	videoPipeline   chan *model.Video
-	needsMetadata   chan *model.Video
-	out             chan *model.Video
-	logger          *slog.Logger
-}
-
-func NewFetch(feedRepo storage.FeedRelRepository, videoRepo storage.VideoRelRepository, channelReader ChannelReader, feedReader FeedReader, interval time.Duration, metadataFetcher MetadataFetcher, logger *slog.Logger) *Fetcher {
-	return &Fetcher{
-		interval:        interval,
-		feedRepo:        feedRepo,
-		videoRepo:       videoRepo,
-		channelReader:   channelReader,
-		feedReader:      feedReader,
-		metadataFetcher: metadataFetcher,
-		feedPipeline:    make(chan *model.Feed, 10),
-		videoPipeline:   make(chan *model.Video, 10),
-		needsMetadata:   make(chan *model.Video, 10),
-		out:             make(chan *model.Video),
-		logger:          logger,
-	}
-}
-
-func (f *Fetcher) Run() {
-	go f.FetchHistoricalVideos()
-	go f.FindNewFeeds()
-
-	go f.ReadFeeds()
-	go f.MetadataFetcher()
-	go f.FindUnprocessed()
-
-	f.logger.Info("started videoPipeline")
-	for {
-		select {
-		case video := <-f.videoPipeline:
-			if err := f.videoRepo.Save(video); err != nil {
-				f.logger.Error("failed to save video in normal db", err)
-				continue
-			}
-			switch video.Status {
-			case model.StatusNew:
-				f.needsMetadata <- video
-			case model.StatusFetched:
-				f.out <- video
-			}
-		}
-	}
-}
-
-func (f *Fetcher) Out() chan *model.Video {
-	return f.out
-}
-
-func (f *Fetcher) FindNewFeeds() {
-	f.logger.Info("looking for new feeds")
-	feeds, err := f.feedRepo.FindByStatus(model.FeedStatusNew)
-	if err != nil {
-		f.logger.Error("failed to fetch feeds", err)
-		return
-	}
-	for _, feed := range feeds {
-		f.feedPipeline <- feed
-	}
-}
-
-func (f *Fetcher) FetchHistoricalVideos() {
-	f.logger.Info("started historical video fetch")
-
-	for feed := range f.feedPipeline {
-		f.logger.Info("fetching historical videos", slog.String("channelid", string(feed.YoutubeChannelID)))
-		token := ""
-		for {
-			token = f.FetchHistoricalVideoPage(feed.YoutubeChannelID, token)
-			if token == "" {
-				break
-			}
-		}
-		feed.Status = model.FeedStatusReady
-		if err := f.feedRepo.Save(feed); err != nil {
-			f.logger.Error("failed to save feed", err)
-			continue
-		}
-	}
-}
-
-func (f *Fetcher) FetchHistoricalVideoPage(channelID model.YoutubeChannelID, pageToken string) string {
-	f.logger.Info("fetching historical video page", slog.String("channelid", string(channelID)), slog.String("pagetoken", pageToken))
-	ytIDs, pageToken, err := f.channelReader.Search(channelID, pageToken)
-	if err != nil {
-		f.logger.Error("failed to fetch channel", err)
-		return ""
-	}
-	for _, ytID := range ytIDs {
-		video := &model.Video{
-			ID:               uuid.New(),
-			Status:           model.StatusNew,
-			YoutubeID:        ytID,
-			YoutubeChannelID: channelID,
-		}
-		if err := f.videoRepo.Save(video); err != nil {
-			f.logger.Error("failed to save video", err)
-			continue
-		}
-		f.videoPipeline <- video
-	}
-
-	f.logger.Info("fetched historical video page", slog.String("channelid", string(channelID)), slog.String("pagetoken", pageToken), slog.Int("count", len(ytIDs)))
-	return pageToken
-}
-
-func (f *Fetcher) FindUnprocessed() {
-	f.logger.Info("looking for unprocessed videos")
-	videos, err := f.videoRepo.FindByStatus(model.StatusNew, model.StatusFetched)
-	if err != nil {
-		f.logger.Error("failed to fetch unprocessed videos", err)
-		return
-	}
-	f.logger.Info("found unprocessed videos", slog.Int("count", len(videos)))
-	for _, video := range videos {
-		f.videoPipeline <- video
-	}
-}
-
-func (f *Fetcher) ReadFeeds() {
-	f.logger.Info("started feed reader")
-	ticker := time.NewTicker(f.interval)
-	for range ticker.C {
-		entries, err := f.feedReader.Unread()
-		if err != nil {
-			f.logger.Error("failed to fetch unread entries", err)
-			continue
-		}
-		f.logger.Info("fetched unread entries", slog.Int("count", len(entries)))
-		if len(entries) == 0 {
-			continue
-		}
-
-		for _, entry := range entries {
-			video := &model.Video{
-				ID:               uuid.New(),
-				Status:           model.StatusNew,
-				YoutubeID:        model.YoutubeVideoID(entry.YoutubeID),
-				YoutubeChannelID: model.YoutubeChannelID(entry.YoutubeChannelID),
-			}
-			if err := f.videoRepo.Save(video); err != nil {
-				f.logger.Error("failed to save video", err)
-				continue
-			}
-			f.videoPipeline <- video
-			if err := f.feedReader.MarkRead(entry.EntryID); err != nil {
-				f.logger.Error("failed to mark entry as read", err)
-				continue
-			}
-		}
-	}
-}
-
-func (f *Fetcher) MetadataFetcher() {
-	f.logger.Info("started metadata fetch")
-
-	buffer := []*model.Video{}
-	timeout := time.NewTimer(10 * time.Second)
-	fetch := make(chan []*model.Video)
-
-	go func() {
-		for videos := range fetch {
-			f.logger.Info("fetching metadata", slog.Int("count", len(videos)))
-			ids := make([]model.YoutubeVideoID, 0, len(videos))
-			for _, video := range videos {
-				ids = append(ids, video.YoutubeID)
-			}
-			mds, err := f.metadataFetcher.FetchMetadata(ids)
-			if err != nil {
-				f.logger.Error("failed to fetch metadata", err)
-				continue
-			}
-			for _, video := range videos {
-				md := mds[video.YoutubeID]
-				video.YoutubeTitle = md.Title
-				video.YoutubeDescription = md.Description
-				video.YoutubeDuration = md.Duration
-				video.YoutubePublishedAt = md.PublishedAt
-				video.Status = model.StatusFetched
-
-				if err := f.videoRepo.Save(video); err != nil {
-					f.logger.Error("failed to save video", err)
-					continue
-				}
-
-				f.out <- video
-			}
-			f.logger.Info("fetched metadata", slog.Int("count", len(videos)))
-		}
-	}()
-
-	for {
-		select {
-		case video := <-f.needsMetadata:
-			timeout.Reset(10 * time.Second)
-			buffer = append(buffer, video)
-			if len(buffer) >= 50 {
-				batch := make([]*model.Video, len(buffer))
-				copy(batch, buffer)
-				fetch <- batch
-				buffer = []*model.Video{}
-			}
-		case <-timeout.C:
-			if len(buffer) == 0 {
-				continue
-			}
-			batch := make([]*model.Video, len(buffer))
-			copy(batch, buffer)
-			fetch <- batch
-			buffer = []*model.Video{}
-		}
-	}
-}

fetch/metadata.go

@@ -1,14 +0,0 @@
-package fetch
-
-import "ewintr.nl/yogai/model"
-
-type Metadata struct {
-	Title       string
-	Description string
-	Duration    string
-	PublishedAt string
-}
-
-type MetadataFetcher interface {
-	FetchMetadata([]model.YoutubeVideoID) (map[model.YoutubeVideoID]Metadata, error)
-}

fetch/youtube.go

@@ -1,76 +0,0 @@
-package fetch
-
-import (
-	"strings"
-
-	"ewintr.nl/yogai/model"
-	"google.golang.org/api/youtube/v3"
-)
-
-type Youtube struct {
-	Client *youtube.Service
-}
-
-func NewYoutube(client *youtube.Service) *Youtube {
-	return &Youtube{Client: client}
-}
-
-func (y *Youtube) Search(channelID model.YoutubeChannelID, pageToken string) ([]model.YoutubeVideoID, string, error) {
-	call := y.Client.Search.
-		List([]string{"id"}).
-		MaxResults(50).
-		Type("video").
-		Order("date").
-		ChannelId(string(channelID))
-
-	if pageToken != "" {
-		call.PageToken(pageToken)
-	}
-
-	response, err := call.Do()
-	if err != nil {
-		return []model.YoutubeVideoID{}, "", err
-	}
-
-	ids := make([]model.YoutubeVideoID, len(response.Items))
-	for i, item := range response.Items {
-		ids[i] = model.YoutubeVideoID(item.Id.VideoId)
-	}
-
-	return ids, response.NextPageToken, nil
-}
-
-func (y *Youtube) FetchMetadata(ytIDs []model.YoutubeVideoID) (map[model.YoutubeVideoID]Metadata, error) {
-	strIDs := make([]string, len(ytIDs))
-	for i, id := range ytIDs {
-		strIDs[i] = string(id)
-	}
-	call := y.Client.Videos.
-		List([]string{"snippet,contentDetails"}).
-		Id(strings.Join(strIDs, ","))
-
-	response, err := call.Do()
-	if err != nil {
-		return map[model.YoutubeVideoID]Metadata{}, err
-	}
-
-	mds := make(map[model.YoutubeVideoID]Metadata, len(response.Items))
-	for _, item := range response.Items {
-		if item.Snippet == nil {
-			continue
-		}
-		md := Metadata{
-			Title:       item.Snippet.Title,
-			Description: item.Snippet.Description,
-			PublishedAt: item.Snippet.PublishedAt,
-		}
-
-		if item.ContentDetails != nil {
-			md.Duration = item.ContentDetails.Duration
-		}
-
-		mds[model.YoutubeVideoID(item.Id)] = md
-	}
-
-	return mds, nil
-}

fetcher/feedreader.go

@@ -0,0 +1,12 @@
+package fetcher
+
+type FeedEntry struct {
+	EntryID   int64
+	FeedID    int64
+	YouTubeID string
+}
+
+type FeedReader interface {
+	Unread() ([]FeedEntry, error)
+	MarkRead(feedID int64) error
+}

fetcher/fetcher.go

@@ -0,0 +1,183 @@
+package fetcher
+
+import (
+	"ewintr.nl/yogai/model"
+	"ewintr.nl/yogai/storage"
+	"github.com/google/uuid"
+	"golang.org/x/exp/slog"
+	"time"
+)
+
+type Fetcher struct {
+	interval        time.Duration
+	videoRepo       storage.VideoRepository
+	feedReader      FeedReader
+	metadataFetcher MetadataFetcher
+	summaryFetcher  SummaryFetcher
+	pipeline        chan *model.Video
+	needsMetadata   chan *model.Video
+	needsSummary    chan *model.Video
+	logger          *slog.Logger
+}
+
+func NewFetch(videoRepo storage.VideoRepository, feedReader FeedReader, interval time.Duration, metadataFetcher MetadataFetcher, summaryFetcher SummaryFetcher, logger *slog.Logger) *Fetcher {
+	return &Fetcher{
+		interval:        interval,
+		videoRepo:       videoRepo,
+		feedReader:      feedReader,
+		metadataFetcher: metadataFetcher,
+		summaryFetcher:  summaryFetcher,
+		pipeline:        make(chan *model.Video, 10),
+		needsMetadata:   make(chan *model.Video, 10),
+		needsSummary:    make(chan *model.Video, 10),
+		logger:          logger,
+	}
+}
+
+func (f *Fetcher) Run() {
+	go f.ReadFeeds()
+	go f.MetadataFetcher()
+	go f.SummaryFetcher()
+	go f.FindUnprocessed()
+
+	f.logger.Info("started pipeline")
+	for {
+		select {
+		case video := <-f.pipeline:
+			switch video.Status {
+			case model.StatusNew:
+				f.needsMetadata <- video
+			case model.StatusHasMetadata:
+				f.needsSummary <- video
+			case model.StatusHasSummary:
+				video.Status = model.StatusReady
+				f.logger.Info("video is ready", slog.String("id", video.ID.String()))
+
+			}
+			if err := f.videoRepo.Save(video); err != nil {
+				f.logger.Error("failed to save video", err)
+				continue
+			}
+		}
+
+	}
+}
+
+func (f *Fetcher) FindUnprocessed() {
+	f.logger.Info("looking for unprocessed videos")
+	videos, err := f.videoRepo.FindByStatus(model.StatusNew, model.StatusHasMetadata)
+	if err != nil {
+		f.logger.Error("failed to fetch unprocessed videos", err)
+		return
+	}
+	f.logger.Info("found unprocessed videos", slog.Int("count", len(videos)))
+	for _, video := range videos {
+		f.pipeline <- video
+	}
+}
+
+func (f *Fetcher) ReadFeeds() {
+	f.logger.Info("started feed reader")
+	ticker := time.NewTicker(f.interval)
+	for range ticker.C {
+		entries, err := f.feedReader.Unread()
+		if err != nil {
+			f.logger.Error("failed to fetch unread entries", err)
+			continue
+		}
+		f.logger.Info("fetched unread entries", slog.Int("count", len(entries)))
+		if len(entries) == 0 {
+			continue
+		}
+
+		for _, entry := range entries {
+			video := &model.Video{
+				ID:        uuid.New(),
+				Status:    model.StatusNew,
+				YoutubeID: entry.YouTubeID,
+				// feed id
+			}
+			if err := f.videoRepo.Save(video); err != nil {
+				f.logger.Error("failed to save video", err)
+				continue
+			}
+			f.pipeline <- video
+			if err := f.feedReader.MarkRead(entry.EntryID); err != nil {
+				f.logger.Error("failed to mark entry as read", err)
+				continue
+			}
+		}
+	}
+}
+
+func (f *Fetcher) MetadataFetcher() {
+	f.logger.Info("started metadata fetcher")
+
+	buffer := []*model.Video{}
+	timeout := time.NewTimer(10 * time.Second)
+	fetch := make(chan []*model.Video)
+
+	go func() {
+		for videos := range fetch {
+			f.logger.Info("fetching metadata", slog.Int("count", len(videos)))
+			ids := make([]string, 0, len(videos))
+			for _, video := range videos {
+				ids = append(ids, video.YoutubeID)
+			}
+			mds, err := f.metadataFetcher.FetchMetadata(ids)
+			if err != nil {
+				f.logger.Error("failed to fetch metadata", err)
+				continue
+			}
+			for _, video := range videos {
+				video.Title = mds[video.YoutubeID].Title
+				video.Description = mds[video.YoutubeID].Description
+				video.Status = model.StatusHasMetadata
+
+				if err := f.videoRepo.Save(video); err != nil {
+					f.logger.Error("failed to save video", err)
+					continue
+				}
+			}
+			f.logger.Info("fetched metadata", slog.Int("count", len(videos)))
+		}
+	}()
+
+	for {
+		select {
+		case video := <-f.needsMetadata:
+			timeout.Reset(10 * time.Second)
+			buffer = append(buffer, video)
+			if len(buffer) >= 10 {
+				batch := make([]*model.Video, len(buffer))
+				copy(batch, buffer)
+				fetch <- batch
+				buffer = []*model.Video{}
+			}
+		case <-timeout.C:
+			if len(buffer) == 0 {
+				continue
+			}
+			batch := make([]*model.Video, len(buffer))
+			copy(batch, buffer)
+			fetch <- batch
+			buffer = []*model.Video{}
+		}
+	}
+}
+
+func (f *Fetcher) SummaryFetcher() {
+	for {
+		select {
+		case video := <-f.needsSummary:
+			f.logger.Info("fetching summary", slog.String("id", video.ID.String()))
+			if err := f.summaryFetcher.FetchSummary(video); err != nil {
+				f.logger.Error("failed to fetch summary", err)
+				continue
+			}
+			video.Status = model.StatusHasSummary
+			f.logger.Info("fetched summary", slog.String("id", video.ID.String()))
+			f.pipeline <- video
+		}
+	}
+}

fetcher/metadata.go

@@ -0,0 +1,10 @@
+package fetcher
+
+type Metadata struct {
+	Title       string
+	Description string
+}
+
+type MetadataFetcher interface {
+	FetchMetadata([]string) (map[string]Metadata, error)
+}

fetcher/miniflux.go

@@ -1,9 +1,8 @@
-package fetch
+package fetcher
 
 import (
-	"strings"
-
 	"miniflux.app/client"
+	"strings"
 )
 
 type Entry struct {
@@ -32,17 +31,24 @@ func NewMiniflux(mflInfo MinifluxInfo) *Miniflux {
 func (m *Miniflux) Unread() ([]FeedEntry, error) {
 	result, err := m.client.Entries(&client.Filter{Status: "unread"})
 	if err != nil {
-		return nil, err
+		return []FeedEntry{}, err
 	}
 
-	entries := make([]FeedEntry, 0, len(result.Entries))
+	entries := []FeedEntry{}
 	for _, entry := range result.Entries {
 		entries = append(entries, FeedEntry{
-			EntryID:          entry.ID,
+			EntryID:   entry.ID,
-			FeedID:           entry.FeedID,
+			FeedID:    entry.FeedID,
-			YoutubeChannelID: strings.TrimPrefix(entry.Feed.FeedURL, "https://www.youtube.com/feeds/videos.xml?channel_id="),
+			YouTubeID: strings.TrimPrefix(entry.URL, "https://www.youtube.com/watch?v="),
-			YoutubeID:        strings.TrimPrefix(entry.URL, "https://www.youtube.com/watch?v="),
 		})
+
+		//	ID:          uuid.New(),
+		//	Status:      model.STATUS_NEW,
+		//	YoutubeURL:  entry.URL,
+		//	FeedID:      strconv.Itoa(int(entry.ID)),
+		//	Title:       entry.Title,
+		//	Description: entry.Content,
+		//})
 	}
 
 	return entries, nil

fetcher/openai.go

@@ -1,34 +1,29 @@
-package process
+package fetcher
 
 import (
 	"context"
-	"fmt"
-
 	"ewintr.nl/yogai/model"
+	"fmt"
 	"github.com/sashabaranov/go-openai"
 )
 
-type OpenAISummarizer struct {
+const summarizePrompt = `You are an helpful assistant. Your task is to extract all text that refers to the content of a yoga workout video from the description a user gives you.
-	client *openai.Client
-}
-
-func NewOpenAISummarizer(client *openai.Client) *OpenAISummarizer {
-	return &OpenAISummarizer{
-		client: client,
-	}
-}
-
-func (sum *OpenAISummarizer) Name() string {
-	return "openai summarizer"
-}
-
-func (sum *OpenAISummarizer) Do(ctx context.Context, video *model.Video) error {
-	const summarizePrompt = `You are an helpful assistant. Your task is to extract all text that refers to the content of a yoga workout video from the description a user gives you.
 You will not add introductory sentences like "This text is about", or "Summary of...". Just give the words verbatim. Trim any white space back to a simple space
 `
 
-	resp, err := sum.client.CreateChatCompletion(
+type OpenAI struct {
-		ctx,
+	client *openai.Client
+}
+
+func NewOpenAI(apiKey string) *OpenAI {
+	return &OpenAI{
+		client: openai.NewClient(apiKey),
+	}
+}
+
+func (o *OpenAI) FetchSummary(video *model.Video) error {
+	resp, err := o.client.CreateChatCompletion(
+		context.Background(),
 		openai.ChatCompletionRequest{
 			Model: openai.GPT4,
 			Messages: []openai.ChatCompletionMessage{
@@ -39,7 +34,7 @@ You will not add introductory sentences like "This text is about", or "Summary o
 
 				{
 					Role:    openai.ChatMessageRoleUser,
-					Content: fmt.Sprintf("%s\n\n%s", video.YoutubeTitle, video.YoutubeDescription),
+					Content: fmt.Sprintf("%s\n\n%s", video.Title, video.Description),
 				},
 			},
 		})

fetcher/summary.go

@@ -1,4 +1,4 @@
-package fetch
+package fetcher
 
 import "ewintr.nl/yogai/model"
 

fetcher/youtube.go

@@ -0,0 +1,35 @@
+package fetcher
+
+import (
+	"google.golang.org/api/youtube/v3"
+	"strings"
+)
+
+type Youtube struct {
+	Client *youtube.Service
+}
+
+func NewYoutube(client *youtube.Service) *Youtube {
+	return &Youtube{Client: client}
+}
+
+func (y *Youtube) FetchMetadata(ytIDs []string) (map[string]Metadata, error) {
+	call := y.Client.Videos.
+		List([]string{"snippet"}).
+		Id(strings.Join(ytIDs, ","))
+
+	response, err := call.Do()
+	if err != nil {
+		return map[string]Metadata{}, err
+	}
+
+	mds := make(map[string]Metadata, len(response.Items))
+	for _, item := range response.Items {
+		mds[item.Id] = Metadata{
+			Title:       item.Snippet.Title,
+			Description: item.Snippet.Description,
+		}
+	}
+
+	return mds, nil
+}

go.mod

@@ -6,8 +6,6 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/lib/pq v1.10.9
 	github.com/sashabaranov/go-openai v1.9.4
-	github.com/weaviate/weaviate v1.19.0
-	github.com/weaviate/weaviate-go-client/v4 v4.8.1
 	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
 	google.golang.org/api v0.122.0
 	miniflux.app v0.0.0-20230505000442-88062ab9f959
@@ -16,28 +14,11 @@ require (
 require (
 	cloud.google.com/go/compute v1.19.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
-	github.com/go-openapi/analysis v0.21.2 // indirect
-	github.com/go-openapi/errors v0.20.3 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.6 // indirect
-	github.com/go-openapi/loads v0.21.1 // indirect
-	github.com/go-openapi/spec v0.20.4 // indirect
-	github.com/go-openapi/strfmt v0.21.3 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
-	github.com/go-openapi/validate v0.21.0 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/s2a-go v0.1.3 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/googleapis/gax-go/v2 v2.8.0 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/oklog/ulid v1.3.1 // indirect
-	go.mongodb.org/mongo-driver v1.11.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/crypto v0.8.0 // indirect
 	golang.org/x/net v0.9.0 // indirect
@@ -48,5 +29,4 @@ require (
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
 	google.golang.org/grpc v1.54.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -7,14 +7,7 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
-github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ=
-github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -24,9 +17,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -35,62 +26,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU=
-github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY=
-github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc=
-github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=
-github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
-github.com/go-openapi/loads v0.21.1 h1:Wb3nVZpdEzDTcly8S4HMkey6fjARRzb7iEaySimlDW0=
-github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g=
-github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=
-github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
-github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg=
-github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k=
-github.com/go-openapi/strfmt v0.21.3 h1:xwhj5X6CjXEZZHMWy1zKJxvW9AfHC9pkyUjLvHtKG7o=
-github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
-github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
-github.com/go-openapi/validate v0.21.0 h1:+Wqk39yKOhfpLqNLEC0/eViCkzM5FVXVqrvt526+wcI=
-github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
-github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
-github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
-github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
-github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
-github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
-github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
-github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
-github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
-github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
-github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
-github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
-github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
-github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
-github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
-github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
-github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
-github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
-github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
-github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
-github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
-github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
-github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -108,19 +46,16 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/s2a-go v0.1.3 h1:FAgZmpLl/SXurPEZyCMPBIiiYeTbqfjlbdnCNTAkbGE=
 github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -129,98 +64,29 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5
 github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc=
 github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
-github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
-github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
-github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
-github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
-github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
-github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
-github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
-github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sashabaranov/go-openai v1.9.4 h1:KanoCEoowAI45jVXlenMCckutSRr39qOmSi9MyPBfZM=
 github.com/sashabaranov/go-openai v1.9.4/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
-github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
-github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/weaviate/weaviate v1.19.0 h1:JKmScZZ5VWVESCkji37bT1cNFCCRIZrne7ENoRHT1vM=
-github.com/weaviate/weaviate v1.19.0/go.mod h1:hvgLEEiZx0gQNEDLNgPGssk8UQjc/CDxZv2Dd5SYgs8=
-github.com/weaviate/weaviate-go-client/v4 v4.8.1 h1:oYU+tS9cRyjB0OLV55oN7pnu+EGfa+yIndo3SVlpWJs=
-github.com/weaviate/weaviate-go-client/v4 v4.8.1/go.mod h1:5vfV3ZVIpG48S18vFi85RR+BFq/YvwN9RCUYoU3oVL0=
-github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
-github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
-github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
-github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
-github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
-go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
-go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8=
-go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y=
-go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
-golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
 golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -241,7 +107,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
@@ -253,25 +118,15 @@ golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -283,7 +138,6 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
@@ -293,11 +147,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -339,20 +189,9 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

handler/handler.go

@@ -1,48 +1 @@
 package handler
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-)
-
-func Index(w http.ResponseWriter) {
-	Message(w, http.StatusOK, "yogai index")
-}
-
-func Message(w http.ResponseWriter, status int, message string, details ...any) {
-	w.WriteHeader(status)
-	response := struct {
-		Message string `json:"message"`
-		Details []any  `json:"details,omitempty"`
-	}{
-		Message: message,
-		Details: details,
-	}
-	body, marshalErr := json.Marshal(response)
-	if marshalErr != nil {
-		fmt.Fprintf(w, fmt.Sprintf(`{"message": %q, "details":%q}`, message, marshalErr.Error()))
-		return
-	}
-	fmt.Fprintf(w, string(body))
-}
-
-func Error(w http.ResponseWriter, status int, message string, err error, details ...any) {
-	w.WriteHeader(status)
-	response := struct {
-		Message string `json:"message"`
-		Error   string `json:"error"`
-		Details []any  `json:"details,omitempty"`
-	}{
-		Message: message,
-		Error:   err.Error(),
-		Details: details,
-	}
-	body, marshalErr := json.Marshal(response)
-	if marshalErr != nil {
-		fmt.Fprintf(w, fmt.Sprintf(`{"message": %q, "error": %q, "details":%q}`, message, err.Error(), marshalErr.Error()))
-		return
-	}
-
-	fmt.Fprintf(w, string(body))
-}

handler/server.go

@@ -1,82 +0,0 @@
-package handler
-
-import (
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"path"
-	"strings"
-
-	"ewintr.nl/yogai/storage"
-	"golang.org/x/exp/slog"
-	"miniflux.app/logger"
-)
-
-type Server struct {
-	apis   map[string]http.Handler
-	logger *slog.Logger
-}
-
-func NewServer(videoRepo storage.VideoRelRepository, logger *slog.Logger) *Server {
-	return &Server{
-		apis: map[string]http.Handler{
-			"video": NewVideoAPI(videoRepo, logger),
-		},
-		logger: logger,
-	}
-}
-
-func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	originalPath := r.URL.Path
-	rec := httptest.NewRecorder() // records the response to be able to mix writing headers and content
-
-	w.Header().Add("Content-Type", "application/json")
-
-	// route to api
-	head, tail := ShiftPath(r.URL.Path)
-	if len(head) == 0 {
-		Index(rec)
-		returnResponse(w, rec)
-		return
-	}
-	api, ok := s.apis[head]
-	if !ok {
-		Error(rec, http.StatusNotFound, "Not found", fmt.Errorf("%s is not a valid path", r.URL.Path))
-	} else {
-		r.URL.Path = tail
-		api.ServeHTTP(rec, r)
-	}
-
-	returnResponse(w, rec)
-	logger.Info("request served", "path", originalPath, "status", rec.Code)
-}
-
-func returnResponse(w http.ResponseWriter, rec *httptest.ResponseRecorder) {
-	w.WriteHeader(rec.Code)
-	for k, v := range rec.Header() {
-		w.Header()[k] = v
-	}
-	w.Write(rec.Body.Bytes())
-}
-
-// ShiftPath splits off the first component of p, which will be cleaned of
-// relative components before processing. head will never contain a slash and
-// tail will always be a rooted path without trailing slash.
-// See https://blog.merovius.de/posts/2017-06-18-how-not-to-use-an-http-router/
-func ShiftPath(p string) (string, string) {
-	p = path.Clean("/" + p)
-
-	// restore iri prefixes that might be mangled by path.Clean
-	for k, v := range map[string]string{
-		"http:/":  "http://",
-		"https:/": "https://",
-	} {
-		p = strings.Replace(p, k, v, -1)
-	}
-
-	i := strings.Index(p[1:], "/") + 1
-	if i <= 0 {
-		return p[1:], "/"
-	}
-	return p[1:i], p[i:]
-}

handler/server.go].go

@@ -0,0 +1 @@
+package handler

handler/video.go

@@ -1,71 +0,0 @@
-package handler
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/http"
-
-	"ewintr.nl/yogai/model"
-	"ewintr.nl/yogai/storage"
-	"golang.org/x/exp/slog"
-)
-
-type VideoAPI struct {
-	videoRepo storage.VideoRelRepository
-	logger    *slog.Logger
-}
-
-func NewVideoAPI(videoRepo storage.VideoRelRepository, logger *slog.Logger) *VideoAPI {
-	return &VideoAPI{
-		videoRepo: videoRepo,
-		logger:    logger,
-	}
-}
-
-func (v *VideoAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	videoID, _ := ShiftPath(r.URL.Path)
-
-	switch {
-	case r.Method == http.MethodGet && videoID == "":
-		v.List(w, r)
-	default:
-		Error(w, http.StatusNotFound, "not found", fmt.Errorf("method %s with subpath %q was not registered in the repository api", r.Method, videoID))
-	}
-}
-
-func (v *VideoAPI) List(w http.ResponseWriter, r *http.Request) {
-	video, err := v.videoRepo.FindByStatus(model.StatusReady)
-	if err != nil {
-		v.returnErr(r.Context(), w, http.StatusInternalServerError, "could not list repositories", err)
-		return
-	}
-
-	type respVideo struct {
-		YoutubeID string `json:"youtube_url"`
-		Title     string `json:"title"`
-		Summary   string `json:"summary"`
-	}
-	var resp []respVideo
-	for _, v := range video {
-		resp = append(resp, respVideo{
-			YoutubeID: string(v.YoutubeID),
-			Title:     v.YoutubeTitle,
-			Summary:   v.Summary,
-		})
-	}
-
-	jsonBody, err := json.Marshal(resp)
-	if err != nil {
-		v.returnErr(r.Context(), w, http.StatusInternalServerError, "could not marshal response", err)
-		return
-	}
-
-	w.WriteHeader(http.StatusOK)
-	fmt.Fprintf(w, string(jsonBody))
-}
-
-func (v *VideoAPI) returnErr(_ context.Context, w http.ResponseWriter, status int, message string, err error, details ...any) {
-	v.logger.Error(message, slog.String("err", err.Error()), slog.String("details", fmt.Sprintf("%+v", details)))
-	Error(w, status, message, err, details...)
-}

model/feed.go

@@ -1,17 +0,0 @@
-package model
-
-import "github.com/google/uuid"
-
-type FeedStatus string
-
-const (
-	FeedStatusNew   FeedStatus = "new"
-	FeedStatusReady FeedStatus = "ready"
-)
-
-type Feed struct {
-	ID               uuid.UUID
-	Status           FeedStatus
-	Title            string
-	YoutubeChannelID YoutubeChannelID
-}

model/video.go

@@ -2,32 +2,21 @@ package model
 
 import "github.com/google/uuid"
 
-type VideoStatus string
+type Status string
 
 const (
-	StatusNew     VideoStatus = "new"
+	StatusNew         Status = "new"
-	StatusFetched VideoStatus = "fetched"
+	StatusHasMetadata Status = "has_metadata"
-	StatusReady   VideoStatus = "ready"
+	StatusHasSummary  Status = "has_summary"
+	StatusReady       Status = "ready"
 )
 
-type YoutubeVideoID string
-
-type YoutubeChannelID string
-
 type Video struct {
-	ID                 uuid.UUID
+	ID          uuid.UUID
-	Status             VideoStatus
+	Status      Status
-	YoutubeID          YoutubeVideoID
+	YoutubeID   string
-	YoutubeChannelID   YoutubeChannelID
+	FeedID      uuid.UUID
-	YoutubeTitle       string
+	Title       string
-	YoutubeDescription string
+	Description string
-	YoutubeDuration    string
+	Summary     string
-	YoutubePublishedAt string
-
-	Summary string
-}
-
-type VideoVec struct {
-	ID      uuid.UUID
-	Summary string
 }

process/processor.go

@@ -1,89 +0,0 @@
-package process
-
-import (
-	"context"
-
-	"ewintr.nl/yogai/model"
-	"ewintr.nl/yogai/storage"
-	"github.com/sashabaranov/go-openai"
-	"golang.org/x/exp/slog"
-)
-
-type VideoProcessor interface {
-	Name() string
-	Do(ctx context.Context, video *model.Video) error
-}
-
-type Processors struct {
-	procs map[string]VideoProcessor
-}
-
-func NewProcessors(openAIClient *openai.Client) *Processors {
-	return &Processors{
-		procs: map[string]VideoProcessor{
-			"summarizer": NewOpenAISummarizer(openAIClient),
-		},
-	}
-}
-
-func (p *Processors) Next(video *model.Video) VideoProcessor {
-	if video.Summary == "" {
-		return p.procs["summarizer"]
-	}
-
-	return nil
-}
-
-type Pipeline struct {
-	in         chan *model.Video
-	procs      *Processors
-	logger     *slog.Logger
-	relStorage storage.VideoRelRepository
-	vecStorage storage.VideoVecRepository
-}
-
-func NewPipeline(in chan *model.Video, processors *Processors, relDB storage.VideoRelRepository, vecDB storage.VideoVecRepository, logger *slog.Logger) *Pipeline {
-	return &Pipeline{
-		in:         in,
-		procs:      processors,
-		relStorage: relDB,
-		vecStorage: vecDB,
-		logger:     logger,
-	}
-}
-
-func (p *Pipeline) Run() {
-	ctx := context.Background()
-	for video := range p.in {
-		p.Process(ctx, video)
-	}
-}
-
-func (p *Pipeline) Process(ctx context.Context, video *model.Video) {
-	p.logger.Info("processing video", slog.String("video", string(video.YoutubeID)))
-	for {
-		next := p.procs.Next(video)
-		if next == nil {
-			p.logger.Info("no more processors for video", slog.String("video", string(video.YoutubeID)))
-			video.Status = model.StatusReady
-			if err := p.relStorage.Save(video); err != nil {
-				p.logger.Error("failed to save video in rel db", slog.String("video", string(video.YoutubeID)), slog.String("error", err.Error()))
-			}
-			return
-		}
-
-		p.logger.Info("processing video", slog.String("video", string(video.YoutubeID)), slog.String("processor", next.Name()))
-		if err := next.Do(context.Background(), video); err != nil {
-			p.logger.Error("failed to process video", slog.String("video", string(video.YoutubeID)), slog.String("processor", next.Name()), slog.String("error", err.Error()))
-			return
-		}
-		if err := p.relStorage.Save(video); err != nil {
-			p.logger.Error("failed to save video in rel db", slog.String("video", string(video.YoutubeID)), slog.String("error", err.Error()))
-			return
-		}
-		if err := p.vecStorage.Save(ctx, video); err != nil {
-			p.logger.Error("failed to save video in rel db", slog.String("video", string(video.YoutubeID)), slog.String("error", err.Error()))
-			return
-		}
-	}
-}

service.go

@@ -2,28 +2,19 @@ package main
 
 import (
 	"context"
-	"fmt"
+	"ewintr.nl/yogai/fetcher"
-	"net/http"
-	"os"
-	"os/signal"
-	"strconv"
-	"time"
-
-	"ewintr.nl/yogai/fetch"
-	"ewintr.nl/yogai/handler"
-	"ewintr.nl/yogai/process"
 	"ewintr.nl/yogai/storage"
-	"github.com/sashabaranov/go-openai"
 	"golang.org/x/exp/slog"
 	"google.golang.org/api/option"
 	"google.golang.org/api/youtube/v3"
+	"os"
+	"os/signal"
+	"time"
 )
 
 func main() {
-
 	ctx := context.Background()
-	logger := slog.New(slog.NewTextHandler(os.Stdout))
+	logger := slog.New(slog.NewTextHandler(os.Stderr))
-
 	postgres, err := storage.NewPostgres(storage.PostgresInfo{
 		Host:     getParam("POSTGRES_HOST", "localhost"),
 		Port:     getParam("POSTGRES_PORT", "5432"),
@@ -35,10 +26,9 @@ func main() {
 		logger.Error("unable to connect to postgres", err)
 		os.Exit(1)
 	}
-	videoRelRepo := storage.NewPostgresVideoRepository(postgres)
+	videoRepo := storage.NewPostgresVideoRepository(postgres)
-	feedRelRepo := storage.NewPostgresFeedRepository(postgres)
 
-	mflxClient := fetch.NewMiniflux(fetch.MinifluxInfo{
+	mflx := fetcher.NewMiniflux(fetcher.MinifluxInfo{
 		Endpoint: getParam("MINIFLUX_ENDPOINT", "http://localhost/v1"),
 		ApiKey:   getParam("MINIFLUX_APIKEY", ""),
 	})
@@ -49,47 +39,18 @@ func main() {
 		os.Exit(1)
 	}
 
-	yt, err := youtube.NewService(ctx, option.WithAPIKey(getParam("YOUTUBE_API_KEY", "")))
+	ytClient, err := youtube.NewService(ctx, option.WithAPIKey(getParam("YOUTUBE_API_KEY", "")))
 	if err != nil {
 		logger.Error("unable to create youtube service", err)
 		os.Exit(1)
 	}
-	ytClient := fetch.NewYoutube(yt)
+	yt := fetcher.NewYoutube(ytClient)
 
-	openaiKey := getParam("OPENAI_API_KEY", "")
+	openAIClient := fetcher.NewOpenAI(getParam("OPENAI_API_KEY", ""))
-	openAIClient := openai.NewClient(openaiKey)
 
-	wvResetSchema := getParam("WEAVIATE_RESET_SCHEMA", "false") == "true"
+	fetcher := fetcher.NewFetch(videoRepo, mflx, fetchInterval, yt, openAIClient, logger)
-	wvClient, err := storage.NewWeaviate(getParam("WEAVIATE_HOST", ""), getParam("WEAVIATE_API_KEY", ""), openaiKey)
-	if err != nil {
-		logger.Error("unable to create weaviate client", err)
-		os.Exit(1)
-	}
-	if wvResetSchema {
-		logger.Info("resetting weaviate schema")
-		if err := wvClient.ResetSchema(); err != nil {
-			logger.Error("unable to reset weaviate schema", err)
-			os.Exit(1)
-		}
-	}
-
-	fetcher := fetch.NewFetch(feedRelRepo, videoRelRepo, ytClient, mflxClient, fetchInterval, ytClient, logger)
 	go fetcher.Run()
-	logger.Info("fetch service started")
+	logger.Info("service started")
-
-	procs := process.NewProcessors(openAIClient)
-	for i := 0; i < 4; i++ {
-		go process.NewPipeline(fetcher.Out(), procs, videoRelRepo, wvClient, logger.With(slog.Int("pipeline", i))).Run()
-	}
-	logger.Info("processing service started")
-
-	port, err := strconv.Atoi(getParam("API_PORT", "8080"))
-	if err != nil {
-		logger.Error("invalid port", err)
-		os.Exit(1)
-	}
-	go http.ListenAndServe(fmt.Sprintf(":%d", port), handler.NewServer(videoRelRepo, logger))
-	logger.Info("http server started")
 
 	done := make(chan os.Signal)
 	signal.Notify(done, os.Interrupt)

storage/migrations.go

@@ -1,55 +0,0 @@
-package storage
-
-var pgMigration = []string{
-	`CREATE TYPE video_status AS ENUM ('new', 'ready')`,
-	`CREATE TABLE video (
-id uuid PRIMARY KEY,
-status video_status NOT NULL,
-youtube_id VARCHAR(255) NOT NULL UNIQUE,
-title VARCHAR(255) NOT NULL,
-feed_id VARCHAR(255) NOT NULL,
-description TEXT,
-summary TEXT
-)`,
-	`CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`,
-	`ALTER TABLE video
-ALTER COLUMN status TYPE video_status_new
-USING video::text::video_status_new`,
-	`DROP TYPE video_status`,
-	`ALTER TYPE video_status_new RENAME TO video_status`,
-	`UPDATE video SET summary = '' WHERE summary IS NULL `,
-	`UPDATE video SET description = '' WHERE description IS NULL `,
-	`ALTER TABLE video 
-ALTER COLUMN summary SET DEFAULT '', 
-ALTER COLUMN summary SET NOT NULL,
-ALTER COLUMN description SET DEFAULT '', 
-ALTER COLUMN description SET NOT NULL`,
-	`CREATE TYPE feed_status AS ENUM ('new', 'ready')`,
-	`CREATE TABLE feed (
-id uuid PRIMARY KEY,
-status feed_status NOT NULL,
-youtube_channel_id VARCHAR(255) NOT NULL UNIQUE,
-title VARCHAR(255) NOT NULL
-)`,
-	`ALTER TABLE video
-DROP COLUMN feed_id,
-ADD COLUMN youtube_channel_id VARCHAR(255) NOT NULL REFERENCES feed(youtube_channel_id)`,
-	`ALTER TABLE video
-ADD COLUMN duration VARCHAR(255),
-ADD COLUMN published_at VARCHAR(255)`,
-	`ALTER TABLE video RENAME COLUMN duration TO youtube_duration`,
-	`ALTER TABLE video RENAME COLUMN published_at TO youtube_published_id`,
-	`ALTER TABLE video RENAME COLUMN title TO youtube_title`,
-	`ALTER TABLE video RENAME COLUMN description TO youtube_description`,
-	`ALTER TABLE video RENAME COLUMN youtube_published_id TO youtube_published_at`,
-	`UPDATE video SET status = 'new'`,
-	`CREATE TYPE video_status_new AS ENUM ('new', 'fetched', 'ready')`,
-	`BEGIN;
-ALTER TABLE video ADD COLUMN status_new video_status_new;
-UPDATE video SET status_new = status::text::video_status_new;
-ALTER TABLE video DROP COLUMN status;
-ALTER TABLE video RENAME COLUMN status_new TO status;
-COMMIT;`,
-	`DROP TYPE video_status`,
-	`ALTER TYPE video_status_new RENAME TO video_status`,
-}

storage/postgres.go

@@ -2,9 +2,8 @@ package storage
 
 import (
 	"database/sql"
-	"fmt"
-
 	"ewintr.nl/yogai/model"
+	"fmt"
 	"github.com/lib/pq"
 	_ "github.com/lib/pq"
 )
@@ -44,26 +43,24 @@ func NewPostgresVideoRepository(postgres *Postgres) *PostgresVideoRepository {
 }
 
 func (p *PostgresVideoRepository) Save(v *model.Video) error {
-	query := `INSERT INTO video (id, status, youtube_id, youtube_channel_id, youtube_title, youtube_description, youtube_duration, youtube_published_at, summary)
+	query := `INSERT INTO video (id, status, youtube_id, feed_id, title, description, summary)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+VALUES ($1, $2, $3, $4, $5, $6, $7)
 ON CONFLICT (id)
 DO UPDATE SET
   id = EXCLUDED.id,
   status = EXCLUDED.status,
   youtube_id = EXCLUDED.youtube_id,
-  youtube_channel_id = EXCLUDED.youtube_channel_id,
+  feed_id = EXCLUDED.feed_id,
-  youtube_title = EXCLUDED.youtube_title,
+  title = EXCLUDED.title,
-  youtube_description = EXCLUDED.youtube_description,
+  description = EXCLUDED.description,
-  youtube_duration = EXCLUDED.youtube_duration,
-  youtube_published_at = EXCLUDED.youtube_published_at,
   summary = EXCLUDED.summary;`
-	_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.YoutubeChannelID, v.YoutubeTitle, v.YoutubeDescription, v.YoutubeDuration, v.YoutubePublishedAt, v.Summary)
+	_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.FeedID, v.Title, v.Description, v.Summary)
 
 	return err
 }
 
-func (p *PostgresVideoRepository) FindByStatus(statuses ...model.VideoStatus) ([]*model.Video, error) {
+func (p *PostgresVideoRepository) FindByStatus(statuses ...model.Status) ([]*model.Video, error) {
-	query := `SELECT id, status, youtube_channel_id, youtube_id, youtube_title, youtube_description,youtube_duration, youtube_published_at, summary
+	query := `SELECT id, status, youtube_id, feed_id, title, description, summary
 FROM video
 WHERE status = ANY($1)`
 	rows, err := p.db.Query(query, pq.Array(statuses))
@@ -74,7 +71,7 @@ WHERE status = ANY($1)`
 	videos := []*model.Video{}
 	for rows.Next() {
 		v := &model.Video{}
-		if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeChannelID, &v.YoutubeID, &v.YoutubeTitle, &v.YoutubeDescription, &v.YoutubeDuration, &v.YoutubePublishedAt, &v.Summary); err != nil {
+		if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeID, &v.FeedID, &v.Title, &v.Description, &v.Summary); err != nil {
 			return nil, err
 		}
 		videos = append(videos, v)
@@ -84,48 +81,30 @@ WHERE status = ANY($1)`
 	return videos, nil
 }
 
-type PostgresFeedRepository struct {
+var pgMigration = []string{
-	*Postgres
+	`CREATE TYPE video_status AS ENUM ('new', 'ready')`,
-}
+	`CREATE TABLE video (
-
+id uuid PRIMARY KEY,
-func NewPostgresFeedRepository(postgres *Postgres) *PostgresFeedRepository {
+status video_status NOT NULL,
-	return &PostgresFeedRepository{postgres}
+youtube_id VARCHAR(255) NOT NULL UNIQUE,
-}
+title VARCHAR(255) NOT NULL,
-
+feed_id VARCHAR(255) NOT NULL,
-func (p *PostgresFeedRepository) Save(f *model.Feed) error {
+description TEXT,
-	query := `INSERT INTO feed (id, status, youtube_channel_id, title)
+summary TEXT
-VALUES ($1, $2, $3, $4)
+)`,
-ON CONFLICT (id)
+	`CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`,
-DO UPDATE SET
+	`ALTER TABLE video
-  id = EXCLUDED.id,
+ALTER COLUMN status TYPE video_status_new
-  status = EXCLUDED.status,
+USING video::text::video_status_new`,
-  youtube_channel_id = EXCLUDED.youtube_channel_id,
+	`DROP TYPE video_status`,
-  title = EXCLUDED.title;`
+	`ALTER TYPE video_status_new RENAME TO video_status`,
-	_, err := p.db.Exec(query, f.ID, f.Status, f.YoutubeChannelID, f.Title)
+	`UPDATE video SET summary = '' WHERE summary IS NULL `,
-
+	`UPDATE video SET description = '' WHERE description IS NULL `,
-	return err
+	`ALTER TABLE video 
-}
+ALTER COLUMN summary SET DEFAULT '', 
-
+ALTER COLUMN summary SET NOT NULL,
-func (p *PostgresFeedRepository) FindByStatus(statuses ...model.FeedStatus) ([]*model.Feed, error) {
+ALTER COLUMN description SET DEFAULT '', 
-	query := `SELECT id, status, youtube_channel_id, title
+ALTER COLUMN description SET NOT NULL`,
-FROM feed
-WHERE status = ANY($1)`
-	rows, err := p.db.Query(query, pq.Array(statuses))
-	if err != nil {
-		return nil, err
-	}
-
-	feeds := []*model.Feed{}
-	for rows.Next() {
-		f := &model.Feed{}
-		if err := rows.Scan(&f.ID, &f.Status, &f.YoutubeChannelID, &f.Title); err != nil {
-			return nil, err
-		}
-		feeds = append(feeds, f)
-	}
-	rows.Close()
-
-	return feeds, nil
 }
 
 func (p *Postgres) migrate(wanted []string) error {

storage/storage.go

@@ -1,21 +1,10 @@
 package storage
 
 import (
-	"context"
-
 	"ewintr.nl/yogai/model"
 )
 
-type FeedRelRepository interface {
+type VideoRepository interface {
-	Save(feed *model.Feed) error
-	FindByStatus(statuses ...model.FeedStatus) ([]*model.Feed, error)
-}
-
-type VideoRelRepository interface {
 	Save(video *model.Video) error
-	FindByStatus(statuses ...model.VideoStatus) ([]*model.Video, error)
+	FindByStatus(statuses ...model.Status) ([]*model.Video, error)
-}
-
-type VideoVecRepository interface {
-	Save(ctx context.Context, video *model.Video) error
 }

storage/weaviate.go

@@ -1,99 +0,0 @@
-package storage
-
-import (
-	"context"
-	"net/http"
-
-	"ewintr.nl/yogai/model"
-	"github.com/weaviate/weaviate-go-client/v4/weaviate"
-	"github.com/weaviate/weaviate-go-client/v4/weaviate/auth"
-	"github.com/weaviate/weaviate-go-client/v4/weaviate/fault"
-	"github.com/weaviate/weaviate/entities/models"
-)
-
-const (
-	className = "Video"
-)
-
-type Weaviate struct {
-	client *weaviate.Client
-}
-
-func NewWeaviate(host, weaviateApiKey, openaiApiKey string) (*Weaviate, error) {
-	config := weaviate.Config{
-		Scheme:     "https",
-		Host:       host,
-		AuthConfig: auth.ApiKey{Value: weaviateApiKey},
-		Headers: map[string]string{
-			"X-OpenAI-Api-Key": openaiApiKey,
-		},
-	}
-
-	c, err := weaviate.NewClient(config)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Weaviate{client: c}, nil
-}
-
-func (w *Weaviate) ResetSchema() error {
-
-	// delete old
-	if err := w.client.Schema().ClassDeleter().WithClassName(className).Do(context.Background()); err != nil {
-		// Weaviate will return a 400 if the class does not exist, so this is allowed, only return an error if it's not a 400
-		if status, ok := err.(*fault.WeaviateClientError); ok && status.StatusCode != http.StatusBadRequest {
-			return err
-		}
-	}
-
-	// create new
-	classObj := &models.Class{
-		Class:      className,
-		Vectorizer: "text2vec-openai",
-		ModuleConfig: map[string]any{
-			"text2vec-openai": map[string]any{
-				"model":        "ada",
-				"modelVersion": "002",
-				"type":         "text",
-			},
-		},
-	}
-
-	return w.client.Schema().ClassCreator().WithClass(classObj).Do(context.Background())
-}
-
-func (w *Weaviate) Save(ctx context.Context, video *model.Video) error {
-	vec := model.VideoVec{
-		ID:      video.ID,
-		Summary: video.Summary,
-	}
-	vID := vec.ID.String()
-	// check it already exists
-	exists, err := w.client.Data().
-		Checker().
-		WithID(vID).
-		WithClassName(className).
-		Do(ctx)
-	if err != nil {
-		return err
-	}
-
-	if exists {
-		return w.client.Data().
-			Updater().
-			WithID(vID).
-			WithClassName(className).
-			WithProperties(vec).
-			Do(ctx)
-	}
-
-	_, err = w.client.Data().
-		Creator().
-		WithClassName(className).
-		WithID(vID).
-		WithProperties(vec).
-		Do(ctx)
-
-	return err
-}