ewintr
/
emdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

finding titles

This commit is contained in:
Erik Winter 2024-03-10 10:21:01 +01:00
parent 2d8fee2c05
commit a861828f47
4 changed files with 29 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
storage/reviewpg.go → storage/review.go
View File

@ -10,6 +10,10 @@ const (
MentionsSeparator = "|"
)
type TitleMentions struct {
Titles []string
}
type ReviewSource string
type Review struct {
@ -20,7 +24,7 @@ type Review struct {
Review string
MovieRating int
Quality int
Titles []string
Mentions TitleMentions
}
type ReviewRepository struct {
@ -34,7 +38,7 @@ func NewReviewRepository(db *Postgres) *ReviewRepository {
}
func (rr *ReviewRepository) Store(r Review) error {
titles, err := json.Marshal(r.Titles)
titles, err := json.Marshal(r.Mentions)
if err != nil {
return err
}
@ -64,7 +68,7 @@ WHERE id=$1`, id)
if err := row.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
return Review{}, err
}
if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
return Review{}, err
}
@ -87,7 +91,7 @@ WHERE movie_id=$1`, movieID)
if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
return []Review{}, err
}
reviews = append(reviews, r)
@ -112,7 +116,7 @@ LIMIT 1`)
if err := row.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
return Review{}, err
}
if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
return Review{}, err
}
@ -135,7 +139,7 @@ WHERE quality=0`)
if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
return []Review{}, err
}
reviews = append(reviews, r)
@ -160,7 +164,7 @@ LIMIT 1`)
if err := row.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
return Review{}, err
}
if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
return Review{}, err
}
@ -183,7 +187,7 @@ WHERE mentioned_titles='{}'`)
if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
return []Review{}, err
}
reviews = append(reviews, r)
@ -208,7 +212,7 @@ FROM review`)
if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
return []Review{}, err
}
reviews = append(reviews, r)

3
worker-client/worker/findalltitles.go
View File

@ -1,8 +1,6 @@
package worker
import (
"time"
"code.ewintr.nl/emdb/job"
)
@ -17,7 +15,6 @@ func (w *Worker) FindAllTitles(jobID int) {
}
for _, r := range reviews {
time.Sleep(1 * time.Second)
if err := w.jq.Add(r.ID, job.ActionFindTitles); err != nil {
logger.Error("could not add job", "error", err)
w.jq.MarkFailed(jobID)

23
worker-client/worker/findtitles.go
View File

@ -3,6 +3,8 @@ package worker
import (
"encoding/json"
"fmt"
"code.ewintr.nl/emdb/storage"
)
const (
@ -12,11 +14,13 @@ const (
%s
----
If you found any movie titles other than %s, list them below in a JSON array. If there are other titles, like TV shows, books or games, ignore them. The format is as follows:
If you found any movie titles other than %s, list them below in a JSON array. If there are titles of other media, like TV shows, books or games, ignore them. The format is as follows:
["movie title 1", "movie title 2"]
---
{"titles": ["movie title 1", "movie title 2"]}
---
Just answer with the JSON and nothing else. If you don't see any other movie titles, just answer with an empty JSON array.`
Just answer with the JSON and nothing else. If you don't see any other movie titles, just use an empty JSON array.`
)
func (w *Worker) FindTitles(jobID int, reviewID string) {
@ -42,23 +46,26 @@ func (w *Worker) FindTitles(jobID int, reviewID string) {
}
prompt := fmt.Sprintf(mentionsTemplate, movieTitle, review.Review, movieTitle)
titles, err := w.ollama.Generate("mistral", prompt)
resp, err := w.ollama.Generate("mistral", prompt)
if err != nil {
logger.Error("could not find titles: %w", err)
}
logger.Info("checked review", "found", titles)
var resp []string
if err := json.Unmarshal([]byte(titles), &resp); err != nil {
logger.Info("checked review", "found", resp)
var mentions storage.TitleMentions
if err := json.Unmarshal([]byte(resp), &mentions); err != nil {
logger.Error("could not unmarshal llm response", "error", err)
w.jq.MarkFailed(jobID)
return
}
review.Titles = resp
review.Mentions = mentions
if err := w.reviewRepo.Store(review); err != nil {
logger.Error("could not update review", "error", err)
w.jq.MarkFailed(jobID)
return
}
logger.Info("done finding title mentions", "count", len(mentions.Titles))
w.jq.MarkDone(jobID)
}

2
worker-client/worker/worker.go
View File

@ -45,7 +45,7 @@ func (w *Worker) Run() {
j, err := w.jq.Next()
switch {
case errors.Is(err, sql.ErrNoRows):
logger.Info("no jobs found")
//logger.Info("no jobs found")
continue
case err != nil:
logger.Error("could not get next job", "error", err)