From a861828f47137e25cda6235107adb1e0c2ae0c63 Mon Sep 17 00:00:00 2001 From: Erik Winter Date: Sun, 10 Mar 2024 10:21:01 +0100 Subject: [PATCH] finding titles --- storage/{reviewpg.go => review.go} | 22 +++++++++++++--------- worker-client/worker/findalltitles.go | 3 --- worker-client/worker/findtitles.go | 23 +++++++++++++++-------- worker-client/worker/worker.go | 2 +- 4 files changed, 29 insertions(+), 21 deletions(-) rename storage/{reviewpg.go => review.go} (89%) diff --git a/storage/reviewpg.go b/storage/review.go similarity index 89% rename from storage/reviewpg.go rename to storage/review.go index 09cad90..cfded6a 100644 --- a/storage/reviewpg.go +++ b/storage/review.go @@ -10,6 +10,10 @@ const ( MentionsSeparator = "|" ) +type TitleMentions struct { + Titles []string +} + type ReviewSource string type Review struct { @@ -20,7 +24,7 @@ type Review struct { Review string MovieRating int Quality int - Titles []string + Mentions TitleMentions } type ReviewRepository struct { @@ -34,7 +38,7 @@ func NewReviewRepository(db *Postgres) *ReviewRepository { } func (rr *ReviewRepository) Store(r Review) error { - titles, err := json.Marshal(r.Titles) + titles, err := json.Marshal(r.Mentions) if err != nil { return err } @@ -64,7 +68,7 @@ WHERE id=$1`, id) if err := row.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil { return Review{}, err } - if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil { + if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil { return Review{}, err } @@ -87,7 +91,7 @@ WHERE movie_id=$1`, movieID) if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil { return nil, err } - if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil { + if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil { return []Review{}, err } reviews = append(reviews, r) @@ -112,7 +116,7 @@ LIMIT 1`) if err := row.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil { return Review{}, err } - if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil { + if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil { return Review{}, err } @@ -135,7 +139,7 @@ WHERE quality=0`) if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil { return nil, err } - if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil { + if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil { return []Review{}, err } reviews = append(reviews, r) @@ -160,7 +164,7 @@ LIMIT 1`) if err := row.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil { return Review{}, err } - if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil { + if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil { return Review{}, err } @@ -183,7 +187,7 @@ WHERE mentioned_titles='{}'`) if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil { return nil, err } - if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil { + if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil { return []Review{}, err } reviews = append(reviews, r) @@ -208,7 +212,7 @@ FROM review`) if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil { return nil, err } - if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil { + if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil { return []Review{}, err } reviews = append(reviews, r) diff --git a/worker-client/worker/findalltitles.go b/worker-client/worker/findalltitles.go index a136881..92b3a49 100644 --- a/worker-client/worker/findalltitles.go +++ b/worker-client/worker/findalltitles.go @@ -1,8 +1,6 @@ package worker import ( - "time" - "code.ewintr.nl/emdb/job" ) @@ -17,7 +15,6 @@ func (w *Worker) FindAllTitles(jobID int) { } for _, r := range reviews { - time.Sleep(1 * time.Second) if err := w.jq.Add(r.ID, job.ActionFindTitles); err != nil { logger.Error("could not add job", "error", err) w.jq.MarkFailed(jobID) diff --git a/worker-client/worker/findtitles.go b/worker-client/worker/findtitles.go index 40ee690..8af7ddf 100644 --- a/worker-client/worker/findtitles.go +++ b/worker-client/worker/findtitles.go @@ -3,6 +3,8 @@ package worker import ( "encoding/json" "fmt" + + "code.ewintr.nl/emdb/storage" ) const ( @@ -12,11 +14,13 @@ const ( %s ---- -If you found any movie titles other than %s, list them below in a JSON array. If there are other titles, like TV shows, books or games, ignore them. The format is as follows: +If you found any movie titles other than %s, list them below in a JSON array. If there are titles of other media, like TV shows, books or games, ignore them. The format is as follows: -["movie title 1", "movie title 2"] +--- +{"titles": ["movie title 1", "movie title 2"]} +--- -Just answer with the JSON and nothing else. If you don't see any other movie titles, just answer with an empty JSON array.` +Just answer with the JSON and nothing else. If you don't see any other movie titles, just use an empty JSON array.` ) func (w *Worker) FindTitles(jobID int, reviewID string) { @@ -42,23 +46,26 @@ func (w *Worker) FindTitles(jobID int, reviewID string) { } prompt := fmt.Sprintf(mentionsTemplate, movieTitle, review.Review, movieTitle) - titles, err := w.ollama.Generate("mistral", prompt) + resp, err := w.ollama.Generate("mistral", prompt) if err != nil { logger.Error("could not find titles: %w", err) } - logger.Info("checked review", "found", titles) - var resp []string - if err := json.Unmarshal([]byte(titles), &resp); err != nil { + logger.Info("checked review", "found", resp) + var mentions storage.TitleMentions + if err := json.Unmarshal([]byte(resp), &mentions); err != nil { logger.Error("could not unmarshal llm response", "error", err) w.jq.MarkFailed(jobID) return } - review.Titles = resp + review.Mentions = mentions if err := w.reviewRepo.Store(review); err != nil { logger.Error("could not update review", "error", err) w.jq.MarkFailed(jobID) return } + + logger.Info("done finding title mentions", "count", len(mentions.Titles)) + w.jq.MarkDone(jobID) } diff --git a/worker-client/worker/worker.go b/worker-client/worker/worker.go index 35ab0ca..abb5e0c 100644 --- a/worker-client/worker/worker.go +++ b/worker-client/worker/worker.go @@ -45,7 +45,7 @@ func (w *Worker) Run() { j, err := w.jq.Next() switch { case errors.Is(err, sql.ErrNoRows): - logger.Info("no jobs found") + //logger.Info("no jobs found") continue case err != nil: logger.Error("could not get next job", "error", err)