diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index e9cf6de..adfc09b 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -197,8 +197,11 @@ func (f *Fetcher) MetadataFetcher() { continue } for _, video := range videos { - video.Title = mds[video.YoutubeID].Title - video.Description = mds[video.YoutubeID].Description + md := mds[video.YoutubeID] + video.YoutubeTitle = md.Title + video.YoutubeDescription = md.Description + video.YoutubeDuration = md.Duration + video.YoutubePublishedAt = md.PublishedAt video.Status = model.StatusHasMetadata if err := f.videoRepo.Save(video); err != nil { diff --git a/fetcher/metadata.go b/fetcher/metadata.go index 182ea9f..187926a 100644 --- a/fetcher/metadata.go +++ b/fetcher/metadata.go @@ -5,6 +5,8 @@ import "ewintr.nl/yogai/model" type Metadata struct { Title string Description string + Duration string + PublishedAt string } type MetadataFetcher interface { diff --git a/fetcher/openai.go b/fetcher/openai.go index c08f114..39219e9 100644 --- a/fetcher/openai.go +++ b/fetcher/openai.go @@ -2,8 +2,9 @@ package fetcher import ( "context" - "ewintr.nl/yogai/model" "fmt" + + "ewintr.nl/yogai/model" "github.com/sashabaranov/go-openai" ) @@ -34,7 +35,7 @@ func (o *OpenAI) FetchSummary(video *model.Video) error { { Role: openai.ChatMessageRoleUser, - Content: fmt.Sprintf("%s\n\n%s", video.Title, video.Description), + Content: fmt.Sprintf("%s\n\n%s", video.YoutubeTitle, video.YoutubeDescription), }, }, }) diff --git a/fetcher/youtube.go b/fetcher/youtube.go index c4883d4..83bb399 100644 --- a/fetcher/youtube.go +++ b/fetcher/youtube.go @@ -46,7 +46,7 @@ func (y *Youtube) FetchMetadata(ytIDs []model.YoutubeVideoID) (map[model.Youtube strIDs[i] = string(id) } call := y.Client.Videos. - List([]string{"snippet"}). + List([]string{"snippet,contentDetails"}). Id(strings.Join(strIDs, ",")) response, err := call.Do() @@ -56,10 +56,20 @@ func (y *Youtube) FetchMetadata(ytIDs []model.YoutubeVideoID) (map[model.Youtube mds := make(map[model.YoutubeVideoID]Metadata, len(response.Items)) for _, item := range response.Items { - mds[model.YoutubeVideoID(item.Id)] = Metadata{ + if item.Snippet == nil { + continue + } + md := Metadata{ Title: item.Snippet.Title, Description: item.Snippet.Description, + PublishedAt: item.Snippet.PublishedAt, } + + if item.ContentDetails != nil { + md.Duration = item.ContentDetails.Duration + } + + mds[model.YoutubeVideoID(item.Id)] = md } return mds, nil diff --git a/handler/video.go b/handler/video.go index 363ff5c..932e5b6 100644 --- a/handler/video.go +++ b/handler/video.go @@ -50,7 +50,7 @@ func (v *VideoAPI) List(w http.ResponseWriter, r *http.Request) { for _, v := range video { resp = append(resp, respVideo{ YoutubeID: string(v.YoutubeID), - Title: v.Title, + Title: v.YoutubeTitle, Summary: v.Summary, }) } diff --git a/model/video.go b/model/video.go index c10fd61..ab17c2c 100644 --- a/model/video.go +++ b/model/video.go @@ -16,11 +16,14 @@ type YoutubeVideoID string type YoutubeChannelID string type Video struct { - ID uuid.UUID - Status VideoStatus - YoutubeID YoutubeVideoID - YoutubeChannelID YoutubeChannelID - Title string - Description string - Summary string + ID uuid.UUID + Status VideoStatus + YoutubeID YoutubeVideoID + YoutubeChannelID YoutubeChannelID + YoutubeTitle string + YoutubeDescription string + YoutubeDuration string + YoutubePublishedAt string + + Summary string } diff --git a/storage/migrations.go b/storage/migrations.go new file mode 100644 index 0000000..9607cd8 --- /dev/null +++ b/storage/migrations.go @@ -0,0 +1,45 @@ +package storage + +var pgMigration = []string{ + `CREATE TYPE video_status AS ENUM ('new', 'ready')`, + `CREATE TABLE video ( +id uuid PRIMARY KEY, +status video_status NOT NULL, +youtube_id VARCHAR(255) NOT NULL UNIQUE, +title VARCHAR(255) NOT NULL, +feed_id VARCHAR(255) NOT NULL, +description TEXT, +summary TEXT +)`, + `CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`, + `ALTER TABLE video +ALTER COLUMN status TYPE video_status_new +USING video::text::video_status_new`, + `DROP TYPE video_status`, + `ALTER TYPE video_status_new RENAME TO video_status`, + `UPDATE video SET summary = '' WHERE summary IS NULL `, + `UPDATE video SET description = '' WHERE description IS NULL `, + `ALTER TABLE video +ALTER COLUMN summary SET DEFAULT '', +ALTER COLUMN summary SET NOT NULL, +ALTER COLUMN description SET DEFAULT '', +ALTER COLUMN description SET NOT NULL`, + `CREATE TYPE feed_status AS ENUM ('new', 'ready')`, + `CREATE TABLE feed ( +id uuid PRIMARY KEY, +status feed_status NOT NULL, +youtube_channel_id VARCHAR(255) NOT NULL UNIQUE, +title VARCHAR(255) NOT NULL +)`, + `ALTER TABLE video +DROP COLUMN feed_id, +ADD COLUMN youtube_channel_id VARCHAR(255) NOT NULL REFERENCES feed(youtube_channel_id)`, + `ALTER TABLE video +ADD COLUMN duration VARCHAR(255), +ADD COLUMN published_at VARCHAR(255)`, + `ALTER TABLE video RENAME COLUMN duration TO youtube_duration`, + `ALTER TABLE video RENAME COLUMN published_at TO youtube_published_id`, + `ALTER TABLE video RENAME COLUMN title TO youtube_title`, + `ALTER TABLE video RENAME COLUMN description TO youtube_description`, + `ALTER TABLE video RENAME COLUMN youtube_published_id TO youtube_published_at`, +} diff --git a/storage/postgres.go b/storage/postgres.go index 39a0f23..f484761 100644 --- a/storage/postgres.go +++ b/storage/postgres.go @@ -44,24 +44,26 @@ func NewPostgresVideoRepository(postgres *Postgres) *PostgresVideoRepository { } func (p *PostgresVideoRepository) Save(v *model.Video) error { - query := `INSERT INTO video (id, status, youtube_id, youtube_channel_id, title, description, summary) -VALUES ($1, $2, $3, $4, $5, $6, $7) + query := `INSERT INTO video (id, status, youtube_id, youtube_channel_id, youtube_title, youtube_description, youtube_duration, youtube_published_at, summary) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (id) DO UPDATE SET id = EXCLUDED.id, status = EXCLUDED.status, youtube_id = EXCLUDED.youtube_id, youtube_channel_id = EXCLUDED.youtube_channel_id, - title = EXCLUDED.title, - description = EXCLUDED.description, + youtube_title = EXCLUDED.youtube_title, + youtube_description = EXCLUDED.youtube_description, + youtube_duration = EXCLUDED.youtube_duration, + youtube_published_at = EXCLUDED.youtube_published_at, summary = EXCLUDED.summary;` - _, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.YoutubeChannelID, v.Title, v.Description, v.Summary) + _, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.YoutubeChannelID, v.YoutubeTitle, v.YoutubeDescription, v.YoutubeDuration, v.YoutubePublishedAt, v.Summary) return err } func (p *PostgresVideoRepository) FindByStatus(statuses ...model.VideoStatus) ([]*model.Video, error) { - query := `SELECT id, status, youtube_channel_id, youtube_id, title, description, summary + query := `SELECT id, status, youtube_channel_id, youtube_id, youtube_title, youtube_description,youtube_duration, youtube_published_at, summary FROM video WHERE status = ANY($1)` rows, err := p.db.Query(query, pq.Array(statuses)) @@ -72,7 +74,7 @@ WHERE status = ANY($1)` videos := []*model.Video{} for rows.Next() { v := &model.Video{} - if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeChannelID, &v.YoutubeID, &v.Title, &v.Description, &v.Summary); err != nil { + if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeChannelID, &v.YoutubeID, &v.YoutubeTitle, &v.YoutubeDescription, &v.YoutubeDuration, &v.YoutubePublishedAt, &v.Summary); err != nil { return nil, err } videos = append(videos, v) @@ -126,42 +128,6 @@ WHERE status = ANY($1)` return feeds, nil } -var pgMigration = []string{ - `CREATE TYPE video_status AS ENUM ('new', 'ready')`, - `CREATE TABLE video ( -id uuid PRIMARY KEY, -status video_status NOT NULL, -youtube_id VARCHAR(255) NOT NULL UNIQUE, -title VARCHAR(255) NOT NULL, -feed_id VARCHAR(255) NOT NULL, -description TEXT, -summary TEXT -)`, - `CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`, - `ALTER TABLE video -ALTER COLUMN status TYPE video_status_new -USING video::text::video_status_new`, - `DROP TYPE video_status`, - `ALTER TYPE video_status_new RENAME TO video_status`, - `UPDATE video SET summary = '' WHERE summary IS NULL `, - `UPDATE video SET description = '' WHERE description IS NULL `, - `ALTER TABLE video -ALTER COLUMN summary SET DEFAULT '', -ALTER COLUMN summary SET NOT NULL, -ALTER COLUMN description SET DEFAULT '', -ALTER COLUMN description SET NOT NULL`, - `CREATE TYPE feed_status AS ENUM ('new', 'ready')`, - `CREATE TABLE feed ( -id uuid PRIMARY KEY, -status feed_status NOT NULL, -youtube_channel_id VARCHAR(255) NOT NULL UNIQUE, -title VARCHAR(255) NOT NULL -)`, - `ALTER TABLE video -DROP COLUMN feed_id, -ADD COLUMN youtube_channel_id VARCHAR(255) NOT NULL REFERENCES feed(youtube_channel_id)`, -} - func (p *Postgres) migrate(wanted []string) error { query := `CREATE TABLE IF NOT EXISTS migration ("id" SERIAL PRIMARY KEY, "query" TEXT)`