more youtube metadata

fetcher/fetcher.go

@@ -197,8 +197,11 @@ func (f *Fetcher) MetadataFetcher() {
 				continue
 			}
 			for _, video := range videos {
-				video.Title = mds[video.YoutubeID].Title
+				md := mds[video.YoutubeID]
-				video.Description = mds[video.YoutubeID].Description
+				video.YoutubeTitle = md.Title
+				video.YoutubeDescription = md.Description
+				video.YoutubeDuration = md.Duration
+				video.YoutubePublishedAt = md.PublishedAt
 				video.Status = model.StatusHasMetadata
 
 				if err := f.videoRepo.Save(video); err != nil {

fetcher/metadata.go

@@ -5,6 +5,8 @@ import "ewintr.nl/yogai/model"
 type Metadata struct {
 	Title       string
 	Description string
+	Duration    string
+	PublishedAt string
 }
 
 type MetadataFetcher interface {

fetcher/openai.go

@@ -2,8 +2,9 @@ package fetcher
 
 import (
 	"context"
-	"ewintr.nl/yogai/model"
 	"fmt"
+
+	"ewintr.nl/yogai/model"
 	"github.com/sashabaranov/go-openai"
 )
 
@@ -34,7 +35,7 @@ func (o *OpenAI) FetchSummary(video *model.Video) error {
 
 				{
 					Role:    openai.ChatMessageRoleUser,
-					Content: fmt.Sprintf("%s\n\n%s", video.Title, video.Description),
+					Content: fmt.Sprintf("%s\n\n%s", video.YoutubeTitle, video.YoutubeDescription),
 				},
 			},
 		})

fetcher/youtube.go

@@ -46,7 +46,7 @@ func (y *Youtube) FetchMetadata(ytIDs []model.YoutubeVideoID) (map[model.Youtube
 		strIDs[i] = string(id)
 	}
 	call := y.Client.Videos.
-		List([]string{"snippet"}).
+		List([]string{"snippet,contentDetails"}).
 		Id(strings.Join(strIDs, ","))
 
 	response, err := call.Do()
@@ -56,10 +56,20 @@ func (y *Youtube) FetchMetadata(ytIDs []model.YoutubeVideoID) (map[model.Youtube
 
 	mds := make(map[model.YoutubeVideoID]Metadata, len(response.Items))
 	for _, item := range response.Items {
-		mds[model.YoutubeVideoID(item.Id)] = Metadata{
+		if item.Snippet == nil {
+			continue
+		}
+		md := Metadata{
 			Title:       item.Snippet.Title,
 			Description: item.Snippet.Description,
+			PublishedAt: item.Snippet.PublishedAt,
 		}
+
+		if item.ContentDetails != nil {
+			md.Duration = item.ContentDetails.Duration
+		}
+
+		mds[model.YoutubeVideoID(item.Id)] = md
 	}
 
 	return mds, nil

handler/video.go

@@ -50,7 +50,7 @@ func (v *VideoAPI) List(w http.ResponseWriter, r *http.Request) {
 	for _, v := range video {
 		resp = append(resp, respVideo{
 			YoutubeID: string(v.YoutubeID),
-			Title:     v.Title,
+			Title:     v.YoutubeTitle,
 			Summary:   v.Summary,
 		})
 	}

model/video.go

@@ -16,11 +16,14 @@ type YoutubeVideoID string
 type YoutubeChannelID string
 
 type Video struct {
-	ID               uuid.UUID
+	ID                 uuid.UUID
-	Status           VideoStatus
+	Status             VideoStatus
-	YoutubeID        YoutubeVideoID
+	YoutubeID          YoutubeVideoID
-	YoutubeChannelID YoutubeChannelID
+	YoutubeChannelID   YoutubeChannelID
-	Title            string
+	YoutubeTitle       string
-	Description      string
+	YoutubeDescription string
-	Summary          string
+	YoutubeDuration    string
+	YoutubePublishedAt string
+
+	Summary string
 }

storage/migrations.go

@@ -0,0 +1,45 @@
+package storage
+
+var pgMigration = []string{
+	`CREATE TYPE video_status AS ENUM ('new', 'ready')`,
+	`CREATE TABLE video (
+id uuid PRIMARY KEY,
+status video_status NOT NULL,
+youtube_id VARCHAR(255) NOT NULL UNIQUE,
+title VARCHAR(255) NOT NULL,
+feed_id VARCHAR(255) NOT NULL,
+description TEXT,
+summary TEXT
+)`,
+	`CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`,
+	`ALTER TABLE video
+ALTER COLUMN status TYPE video_status_new
+USING video::text::video_status_new`,
+	`DROP TYPE video_status`,
+	`ALTER TYPE video_status_new RENAME TO video_status`,
+	`UPDATE video SET summary = '' WHERE summary IS NULL `,
+	`UPDATE video SET description = '' WHERE description IS NULL `,
+	`ALTER TABLE video 
+ALTER COLUMN summary SET DEFAULT '', 
+ALTER COLUMN summary SET NOT NULL,
+ALTER COLUMN description SET DEFAULT '', 
+ALTER COLUMN description SET NOT NULL`,
+	`CREATE TYPE feed_status AS ENUM ('new', 'ready')`,
+	`CREATE TABLE feed (
+id uuid PRIMARY KEY,
+status feed_status NOT NULL,
+youtube_channel_id VARCHAR(255) NOT NULL UNIQUE,
+title VARCHAR(255) NOT NULL
+)`,
+	`ALTER TABLE video
+DROP COLUMN feed_id,
+ADD COLUMN youtube_channel_id VARCHAR(255) NOT NULL REFERENCES feed(youtube_channel_id)`,
+	`ALTER TABLE video
+ADD COLUMN duration VARCHAR(255),
+ADD COLUMN published_at VARCHAR(255)`,
+	`ALTER TABLE video RENAME COLUMN duration TO youtube_duration`,
+	`ALTER TABLE video RENAME COLUMN published_at TO youtube_published_id`,
+	`ALTER TABLE video RENAME COLUMN title TO youtube_title`,
+	`ALTER TABLE video RENAME COLUMN description TO youtube_description`,
+	`ALTER TABLE video RENAME COLUMN youtube_published_id TO youtube_published_at`,
+}

storage/postgres.go

@@ -44,24 +44,26 @@ func NewPostgresVideoRepository(postgres *Postgres) *PostgresVideoRepository {
 }
 
 func (p *PostgresVideoRepository) Save(v *model.Video) error {
-	query := `INSERT INTO video (id, status, youtube_id, youtube_channel_id, title, description, summary)
+	query := `INSERT INTO video (id, status, youtube_id, youtube_channel_id, youtube_title, youtube_description, youtube_duration, youtube_published_at, summary)
-VALUES ($1, $2, $3, $4, $5, $6, $7)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
 ON CONFLICT (id)
 DO UPDATE SET
   id = EXCLUDED.id,
   status = EXCLUDED.status,
   youtube_id = EXCLUDED.youtube_id,
   youtube_channel_id = EXCLUDED.youtube_channel_id,
-  title = EXCLUDED.title,
+  youtube_title = EXCLUDED.youtube_title,
-  description = EXCLUDED.description,
+  youtube_description = EXCLUDED.youtube_description,
+  youtube_duration = EXCLUDED.youtube_duration,
+  youtube_published_at = EXCLUDED.youtube_published_at,
   summary = EXCLUDED.summary;`
-	_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.YoutubeChannelID, v.Title, v.Description, v.Summary)
+	_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.YoutubeChannelID, v.YoutubeTitle, v.YoutubeDescription, v.YoutubeDuration, v.YoutubePublishedAt, v.Summary)
 
 	return err
 }
 
 func (p *PostgresVideoRepository) FindByStatus(statuses ...model.VideoStatus) ([]*model.Video, error) {
-	query := `SELECT id, status, youtube_channel_id, youtube_id, title, description, summary
+	query := `SELECT id, status, youtube_channel_id, youtube_id, youtube_title, youtube_description,youtube_duration, youtube_published_at, summary
 FROM video
 WHERE status = ANY($1)`
 	rows, err := p.db.Query(query, pq.Array(statuses))
@@ -72,7 +74,7 @@ WHERE status = ANY($1)`
 	videos := []*model.Video{}
 	for rows.Next() {
 		v := &model.Video{}
-		if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeChannelID, &v.YoutubeID, &v.Title, &v.Description, &v.Summary); err != nil {
+		if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeChannelID, &v.YoutubeID, &v.YoutubeTitle, &v.YoutubeDescription, &v.YoutubeDuration, &v.YoutubePublishedAt, &v.Summary); err != nil {
 			return nil, err
 		}
 		videos = append(videos, v)
@@ -126,42 +128,6 @@ WHERE status = ANY($1)`
 	return feeds, nil
 }
 
-var pgMigration = []string{
-	`CREATE TYPE video_status AS ENUM ('new', 'ready')`,
-	`CREATE TABLE video (
-id uuid PRIMARY KEY,
-status video_status NOT NULL,
-youtube_id VARCHAR(255) NOT NULL UNIQUE,
-title VARCHAR(255) NOT NULL,
-feed_id VARCHAR(255) NOT NULL,
-description TEXT,
-summary TEXT
-)`,
-	`CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`,
-	`ALTER TABLE video
-ALTER COLUMN status TYPE video_status_new
-USING video::text::video_status_new`,
-	`DROP TYPE video_status`,
-	`ALTER TYPE video_status_new RENAME TO video_status`,
-	`UPDATE video SET summary = '' WHERE summary IS NULL `,
-	`UPDATE video SET description = '' WHERE description IS NULL `,
-	`ALTER TABLE video 
-ALTER COLUMN summary SET DEFAULT '', 
-ALTER COLUMN summary SET NOT NULL,
-ALTER COLUMN description SET DEFAULT '', 
-ALTER COLUMN description SET NOT NULL`,
-	`CREATE TYPE feed_status AS ENUM ('new', 'ready')`,
-	`CREATE TABLE feed (
-id uuid PRIMARY KEY,
-status feed_status NOT NULL,
-youtube_channel_id VARCHAR(255) NOT NULL UNIQUE,
-title VARCHAR(255) NOT NULL
-)`,
-	`ALTER TABLE video
-DROP COLUMN feed_id,
-ADD COLUMN youtube_channel_id VARCHAR(255) NOT NULL REFERENCES feed(youtube_channel_id)`,
-}
-
 func (p *Postgres) migrate(wanted []string) error {
 	query := `CREATE TABLE IF NOT EXISTS migration
 ("id" SERIAL PRIMARY KEY, "query" TEXT)`