ewintr
/
yogai
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more youtube metadata

This commit is contained in:
Erik Winter 2023-05-31 16:27:35 +02:00
parent b8c10a9d58
commit 82d95e98c9
8 changed files with 87 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
fetcher/fetcher.go
View File

@ -197,8 +197,11 @@ func (f *Fetcher) MetadataFetcher() {
continue
}
for _, video := range videos {
video.Title = mds[video.YoutubeID].Title
video.Description = mds[video.YoutubeID].Description
md := mds[video.YoutubeID]
video.YoutubeTitle = md.Title
video.YoutubeDescription = md.Description
video.YoutubeDuration = md.Duration
video.YoutubePublishedAt = md.PublishedAt
video.Status = model.StatusHasMetadata
if err := f.videoRepo.Save(video); err != nil {

2
fetcher/metadata.go
View File

@ -5,6 +5,8 @@ import "ewintr.nl/yogai/model"
type Metadata struct {
Title string
Description string
Duration string
PublishedAt string
}
type MetadataFetcher interface {

5
fetcher/openai.go
View File

@ -2,8 +2,9 @@ package fetcher
import (
"context"
"ewintr.nl/yogai/model"
"fmt"
"ewintr.nl/yogai/model"
"github.com/sashabaranov/go-openai"
)
@ -34,7 +35,7 @@ func (o *OpenAI) FetchSummary(video *model.Video) error {
{
Role: openai.ChatMessageRoleUser,
Content: fmt.Sprintf("%s\n\n%s", video.Title, video.Description),
Content: fmt.Sprintf("%s\n\n%s", video.YoutubeTitle, video.YoutubeDescription),
},
},
})

14
fetcher/youtube.go
View File

@ -46,7 +46,7 @@ func (y *Youtube) FetchMetadata(ytIDs []model.YoutubeVideoID) (map[model.Youtube
strIDs[i] = string(id)
}
call := y.Client.Videos.
List([]string{"snippet"}).
List([]string{"snippet,contentDetails"}).
Id(strings.Join(strIDs, ","))
response, err := call.Do()
@ -56,10 +56,20 @@ func (y *Youtube) FetchMetadata(ytIDs []model.YoutubeVideoID) (map[model.Youtube
mds := make(map[model.YoutubeVideoID]Metadata, len(response.Items))
for _, item := range response.Items {
mds[model.YoutubeVideoID(item.Id)] = Metadata{
if item.Snippet == nil {
continue
}
md := Metadata{
Title: item.Snippet.Title,
Description: item.Snippet.Description,
PublishedAt: item.Snippet.PublishedAt,
}
if item.ContentDetails != nil {
md.Duration = item.ContentDetails.Duration
}
mds[model.YoutubeVideoID(item.Id)] = md
}
return mds, nil

2
handler/video.go
View File

@ -50,7 +50,7 @@ func (v *VideoAPI) List(w http.ResponseWriter, r *http.Request) {
for _, v := range video {
resp = append(resp, respVideo{
YoutubeID: string(v.YoutubeID),
Title: v.Title,
Title: v.YoutubeTitle,
Summary: v.Summary,
})
}

7
model/video.go
View File

@ -20,7 +20,10 @@ type Video struct {
Status VideoStatus
YoutubeID YoutubeVideoID
YoutubeChannelID YoutubeChannelID
Title string
Description string
YoutubeTitle string
YoutubeDescription string
YoutubeDuration string
YoutubePublishedAt string
Summary string
}

45
storage/migrations.go Normal file
View File

@ -0,0 +1,45 @@
package storage
var pgMigration = []string{
`CREATE TYPE video_status AS ENUM ('new', 'ready')`,
`CREATE TABLE video (
id uuid PRIMARY KEY,
status video_status NOT NULL,
youtube_id VARCHAR(255) NOT NULL UNIQUE,
title VARCHAR(255) NOT NULL,
feed_id VARCHAR(255) NOT NULL,
description TEXT,
summary TEXT
)`,
`CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`,
`ALTER TABLE video
ALTER COLUMN status TYPE video_status_new
USING video::text::video_status_new`,
`DROP TYPE video_status`,
`ALTER TYPE video_status_new RENAME TO video_status`,
`UPDATE video SET summary = '' WHERE summary IS NULL `,
`UPDATE video SET description = '' WHERE description IS NULL `,
`ALTER TABLE video
ALTER COLUMN summary SET DEFAULT '',
ALTER COLUMN summary SET NOT NULL,
ALTER COLUMN description SET DEFAULT '',
ALTER COLUMN description SET NOT NULL`,
`CREATE TYPE feed_status AS ENUM ('new', 'ready')`,
`CREATE TABLE feed (
id uuid PRIMARY KEY,
status feed_status NOT NULL,
youtube_channel_id VARCHAR(255) NOT NULL UNIQUE,
title VARCHAR(255) NOT NULL
)`,
`ALTER TABLE video
DROP COLUMN feed_id,
ADD COLUMN youtube_channel_id VARCHAR(255) NOT NULL REFERENCES feed(youtube_channel_id)`,
`ALTER TABLE video
ADD COLUMN duration VARCHAR(255),
ADD COLUMN published_at VARCHAR(255)`,
`ALTER TABLE video RENAME COLUMN duration TO youtube_duration`,
`ALTER TABLE video RENAME COLUMN published_at TO youtube_published_id`,
`ALTER TABLE video RENAME COLUMN title TO youtube_title`,
`ALTER TABLE video RENAME COLUMN description TO youtube_description`,
`ALTER TABLE video RENAME COLUMN youtube_published_id TO youtube_published_at`,
}

52
storage/postgres.go
View File

@ -44,24 +44,26 @@ func NewPostgresVideoRepository(postgres *Postgres) *PostgresVideoRepository {
}
func (p *PostgresVideoRepository) Save(v *model.Video) error {
query := `INSERT INTO video (id, status, youtube_id, youtube_channel_id, title, description, summary)
VALUES ($1, $2, $3, $4, $5, $6, $7)
query := `INSERT INTO video (id, status, youtube_id, youtube_channel_id, youtube_title, youtube_description, youtube_duration, youtube_published_at, summary)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (id)
DO UPDATE SET
id = EXCLUDED.id,
status = EXCLUDED.status,
youtube_id = EXCLUDED.youtube_id,
youtube_channel_id = EXCLUDED.youtube_channel_id,
title = EXCLUDED.title,
description = EXCLUDED.description,
youtube_title = EXCLUDED.youtube_title,
youtube_description = EXCLUDED.youtube_description,
youtube_duration = EXCLUDED.youtube_duration,
youtube_published_at = EXCLUDED.youtube_published_at,
summary = EXCLUDED.summary;`
_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.YoutubeChannelID, v.Title, v.Description, v.Summary)
_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.YoutubeChannelID, v.YoutubeTitle, v.YoutubeDescription, v.YoutubeDuration, v.YoutubePublishedAt, v.Summary)
return err
}
func (p *PostgresVideoRepository) FindByStatus(statuses ...model.VideoStatus) ([]*model.Video, error) {
query := `SELECT id, status, youtube_channel_id, youtube_id, title, description, summary
query := `SELECT id, status, youtube_channel_id, youtube_id, youtube_title, youtube_description,youtube_duration, youtube_published_at, summary
FROM video
WHERE status = ANY($1)`
rows, err := p.db.Query(query, pq.Array(statuses))
@ -72,7 +74,7 @@ WHERE status = ANY($1)`
videos := []*model.Video{}
for rows.Next() {
v := &model.Video{}
if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeChannelID, &v.YoutubeID, &v.Title, &v.Description, &v.Summary); err != nil {
if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeChannelID, &v.YoutubeID, &v.YoutubeTitle, &v.YoutubeDescription, &v.YoutubeDuration, &v.YoutubePublishedAt, &v.Summary); err != nil {
return nil, err
}
videos = append(videos, v)
@ -126,42 +128,6 @@ WHERE status = ANY($1)`
return feeds, nil
}
var pgMigration = []string{
`CREATE TYPE video_status AS ENUM ('new', 'ready')`,
`CREATE TABLE video (
id uuid PRIMARY KEY,
status video_status NOT NULL,
youtube_id VARCHAR(255) NOT NULL UNIQUE,
title VARCHAR(255) NOT NULL,
feed_id VARCHAR(255) NOT NULL,
description TEXT,
summary TEXT
)`,
`CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`,
`ALTER TABLE video
ALTER COLUMN status TYPE video_status_new
USING video::text::video_status_new`,
`DROP TYPE video_status`,
`ALTER TYPE video_status_new RENAME TO video_status`,
`UPDATE video SET summary = '' WHERE summary IS NULL `,
`UPDATE video SET description = '' WHERE description IS NULL `,
`ALTER TABLE video
ALTER COLUMN summary SET DEFAULT '',
ALTER COLUMN summary SET NOT NULL,
ALTER COLUMN description SET DEFAULT '',
ALTER COLUMN description SET NOT NULL`,
`CREATE TYPE feed_status AS ENUM ('new', 'ready')`,
`CREATE TABLE feed (
id uuid PRIMARY KEY,
status feed_status NOT NULL,
youtube_channel_id VARCHAR(255) NOT NULL UNIQUE,
title VARCHAR(255) NOT NULL
)`,
`ALTER TABLE video
DROP COLUMN feed_id,
ADD COLUMN youtube_channel_id VARCHAR(255) NOT NULL REFERENCES feed(youtube_channel_id)`,
}
func (p *Postgres) migrate(wanted []string) error {
query := `CREATE TABLE IF NOT EXISTS migration
("id" SERIAL PRIMARY KEY, "query" TEXT)`