ewintr
/
yogai
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more youtube metadata

This commit is contained in:
Erik Winter 2023-05-31 16:27:35 +02:00
parent b8c10a9d58
commit 82d95e98c9
8 changed files with 87 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
fetcher/fetcher.go
View File

@ -197,8 +197,11 @@ func (f *Fetcher) MetadataFetcher() {
continue
}
for _, video := range videos {
video.Title = mds[video.YoutubeID].Title
video.Description = mds[video.YoutubeID].Description
md := mds[video.YoutubeID]
video.YoutubeTitle = md.Title
video.YoutubeDescription = md.Description
video.YoutubeDuration = md.Duration
video.YoutubePublishedAt = md.PublishedAt
video.Status = model.StatusHasMetadata
if err := f.videoRepo.Save(video); err != nil {

2
fetcher/metadata.go
View File

@ -5,6 +5,8 @@ import "ewintr.nl/yogai/model"
type Metadata struct {
Title string
Description string
Duration string
PublishedAt string
}
type MetadataFetcher interface {

5
fetcher/openai.go
View File

@ -2,8 +2,9 @@ package fetcher
import (
"context"
"ewintr.nl/yogai/model"
"fmt"
"ewintr.nl/yogai/model"
"github.com/sashabaranov/go-openai"
)
@ -34,7 +35,7 @@ func (o *OpenAI) FetchSummary(video *model.Video) error {
{
Role: openai.ChatMessageRoleUser,
Content: fmt.Sprintf("%s\n\n%s", video.Title, video.Description),
Content: fmt.Sprintf("%s\n\n%s", video.YoutubeTitle, video.YoutubeDescription),
},
},
})

14
fetcher/youtube.go
View File

@ -46,7 +46,7 @@ func (y *Youtube) FetchMetadata(ytIDs []model.YoutubeVideoID) (map[model.Youtube
strIDs[i] = string(id)
}
call := y.Client.Videos.
List([]string{"snippet"}).
List([]string{"snippet,contentDetails"}).
Id(strings.Join(strIDs, ","))
response, err := call.Do()
@ -56,10 +56,20 @@ func (y *Youtube) FetchMetadata(ytIDs []model.YoutubeVideoID) (map[model.Youtube
mds := make(map[model.YoutubeVideoID]Metadata, len(response.Items))
for _, item := range response.Items {
mds[model.YoutubeVideoID(item.Id)] = Metadata{
if item.Snippet == nil {
continue
}
md := Metadata{
Title: item.Snippet.Title,
Description: item.Snippet.Description,
PublishedAt: item.Snippet.PublishedAt,
}
if item.ContentDetails != nil {
md.Duration = item.ContentDetails.Duration
}
mds[model.YoutubeVideoID(item.Id)] = md
}
return mds, nil

2
handler/video.go
View File

@ -50,7 +50,7 @@ func (v *VideoAPI) List(w http.ResponseWriter, r *http.Request) {
for _, v := range video {
resp = append(resp, respVideo{
YoutubeID: string(v.YoutubeID),
Title: v.Title,
Title: v.YoutubeTitle,
Summary: v.Summary,
})
}

17
model/video.go
View File

@ -16,11 +16,14 @@ type YoutubeVideoID string
type YoutubeChannelID string
type Video struct {
ID uuid.UUID
Status VideoStatus
YoutubeID YoutubeVideoID
YoutubeChannelID YoutubeChannelID
Title string
Description string
Summary string
ID uuid.UUID
Status VideoStatus
YoutubeID YoutubeVideoID
YoutubeChannelID YoutubeChannelID
YoutubeTitle string
YoutubeDescription string
YoutubeDuration string
YoutubePublishedAt string
Summary string
}

45
storage/migrations.go Normal file
View File

@ -0,0 +1,45 @@
package storage
var pgMigration = []string{
`CREATE TYPE video_status AS ENUM ('new', 'ready')`,
`CREATE TABLE video (
id uuid PRIMARY KEY,
status video_status NOT NULL,
youtube_id VARCHAR(255) NOT NULL UNIQUE,
title VARCHAR(255) NOT NULL,
feed_id VARCHAR(255) NOT NULL,
description TEXT,
summary TEXT
)`,
`CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`,
`ALTER TABLE video
ALTER COLUMN status TYPE video_status_new
USING video::text::video_status_new`,
`DROP TYPE video_status`,
`ALTER TYPE video_status_new RENAME TO video_status`,
`UPDATE video SET summary = '' WHERE summary IS NULL `,
`UPDATE video SET description = '' WHERE description IS NULL `,
`ALTER TABLE video
ALTER COLUMN summary SET DEFAULT '',
ALTER COLUMN summary SET NOT NULL,
ALTER COLUMN description SET DEFAULT '',
ALTER COLUMN description SET NOT NULL`,
`CREATE TYPE feed_status AS ENUM ('new', 'ready')`,
`CREATE TABLE feed (
id uuid PRIMARY KEY,
status feed_status NOT NULL,
youtube_channel_id VARCHAR(255) NOT NULL UNIQUE,
title VARCHAR(255) NOT NULL
)`,
`ALTER TABLE video
DROP COLUMN feed_id,
ADD COLUMN youtube_channel_id VARCHAR(255) NOT NULL REFERENCES feed(youtube_channel_id)`,
`ALTER TABLE video
ADD COLUMN duration VARCHAR(255),
ADD COLUMN published_at VARCHAR(255)`,
`ALTER TABLE video RENAME COLUMN duration TO youtube_duration`,
`ALTER TABLE video RENAME COLUMN published_at TO youtube_published_id`,
`ALTER TABLE video RENAME COLUMN title TO youtube_title`,
`ALTER TABLE video RENAME COLUMN description TO youtube_description`,
`ALTER TABLE video RENAME COLUMN youtube_published_id TO youtube_published_at`,
}

52
storage/postgres.go
View File

@ -44,24 +44,26 @@ func NewPostgresVideoRepository(postgres *Postgres) *PostgresVideoRepository {
}
func (p *PostgresVideoRepository) Save(v *model.Video) error {
query := `INSERT INTO video (id, status, youtube_id, youtube_channel_id, title, description, summary)
VALUES ($1, $2, $3, $4, $5, $6, $7)
query := `INSERT INTO video (id, status, youtube_id, youtube_channel_id, youtube_title, youtube_description, youtube_duration, youtube_published_at, summary)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (id)
DO UPDATE SET
id = EXCLUDED.id,
status = EXCLUDED.status,
youtube_id = EXCLUDED.youtube_id,
youtube_channel_id = EXCLUDED.youtube_channel_id,
title = EXCLUDED.title,
description = EXCLUDED.description,
youtube_title = EXCLUDED.youtube_title,
youtube_description = EXCLUDED.youtube_description,
youtube_duration = EXCLUDED.youtube_duration,
youtube_published_at = EXCLUDED.youtube_published_at,
summary = EXCLUDED.summary;`
_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.YoutubeChannelID, v.Title, v.Description, v.Summary)
_, err := p.db.Exec(query, v.ID, v.Status, v.YoutubeID, v.YoutubeChannelID, v.YoutubeTitle, v.YoutubeDescription, v.YoutubeDuration, v.YoutubePublishedAt, v.Summary)
return err
}
func (p *PostgresVideoRepository) FindByStatus(statuses ...model.VideoStatus) ([]*model.Video, error) {
query := `SELECT id, status, youtube_channel_id, youtube_id, title, description, summary
query := `SELECT id, status, youtube_channel_id, youtube_id, youtube_title, youtube_description,youtube_duration, youtube_published_at, summary
FROM video
WHERE status = ANY($1)`
rows, err := p.db.Query(query, pq.Array(statuses))
@ -72,7 +74,7 @@ WHERE status = ANY($1)`
videos := []*model.Video{}
for rows.Next() {
v := &model.Video{}
if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeChannelID, &v.YoutubeID, &v.Title, &v.Description, &v.Summary); err != nil {
if err := rows.Scan(&v.ID, &v.Status, &v.YoutubeChannelID, &v.YoutubeID, &v.YoutubeTitle, &v.YoutubeDescription, &v.YoutubeDuration, &v.YoutubePublishedAt, &v.Summary); err != nil {
return nil, err
}
videos = append(videos, v)
@ -126,42 +128,6 @@ WHERE status = ANY($1)`
return feeds, nil
}
var pgMigration = []string{
`CREATE TYPE video_status AS ENUM ('new', 'ready')`,
`CREATE TABLE video (
id uuid PRIMARY KEY,
status video_status NOT NULL,
youtube_id VARCHAR(255) NOT NULL UNIQUE,
title VARCHAR(255) NOT NULL,
feed_id VARCHAR(255) NOT NULL,
description TEXT,
summary TEXT
)`,
`CREATE TYPE video_status_new AS ENUM ('new', 'has_metadata', 'has_summary', 'ready')`,
`ALTER TABLE video
ALTER COLUMN status TYPE video_status_new
USING video::text::video_status_new`,
`DROP TYPE video_status`,
`ALTER TYPE video_status_new RENAME TO video_status`,
`UPDATE video SET summary = '' WHERE summary IS NULL `,
`UPDATE video SET description = '' WHERE description IS NULL `,
`ALTER TABLE video
ALTER COLUMN summary SET DEFAULT '',
ALTER COLUMN summary SET NOT NULL,
ALTER COLUMN description SET DEFAULT '',
ALTER COLUMN description SET NOT NULL`,
`CREATE TYPE feed_status AS ENUM ('new', 'ready')`,
`CREATE TABLE feed (
id uuid PRIMARY KEY,
status feed_status NOT NULL,
youtube_channel_id VARCHAR(255) NOT NULL UNIQUE,
title VARCHAR(255) NOT NULL
)`,
`ALTER TABLE video
DROP COLUMN feed_id,
ADD COLUMN youtube_channel_id VARCHAR(255) NOT NULL REFERENCES feed(youtube_channel_id)`,
}
func (p *Postgres) migrate(wanted []string) error {
query := `CREATE TABLE IF NOT EXISTS migration
("id" SERIAL PRIMARY KEY, "query" TEXT)`