finding titles

storage/review.go

@@ -10,6 +10,10 @@ const (
 	MentionsSeparator = "|"
 )
 
+type TitleMentions struct {
+	Titles []string
+}
+
 type ReviewSource string
 
 type Review struct {
@@ -20,7 +24,7 @@ type Review struct {
 	Review      string
 	MovieRating int
 	Quality     int
-	Titles      []string
+	Mentions    TitleMentions
 }
 
 type ReviewRepository struct {
@@ -34,7 +38,7 @@ func NewReviewRepository(db *Postgres) *ReviewRepository {
 }
 
 func (rr *ReviewRepository) Store(r Review) error {
-	titles, err := json.Marshal(r.Titles)
+	titles, err := json.Marshal(r.Mentions)
 	if err != nil {
 		return err
 	}
@@ -64,7 +68,7 @@ WHERE id=$1`, id)
 	if err := row.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
 		return Review{}, err
 	}
-	if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
+	if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
 		return Review{}, err
 	}
 
@@ -87,7 +91,7 @@ WHERE movie_id=$1`, movieID)
 		if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
 			return nil, err
 		}
-		if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
+		if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
 			return []Review{}, err
 		}
 		reviews = append(reviews, r)
@@ -112,7 +116,7 @@ LIMIT 1`)
 	if err := row.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
 		return Review{}, err
 	}
-	if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
+	if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
 		return Review{}, err
 	}
 
@@ -135,7 +139,7 @@ WHERE quality=0`)
 		if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
 			return nil, err
 		}
-		if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
+		if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
 			return []Review{}, err
 		}
 		reviews = append(reviews, r)
@@ -160,7 +164,7 @@ LIMIT 1`)
 	if err := row.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
 		return Review{}, err
 	}
-	if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
+	if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
 		return Review{}, err
 	}
 
@@ -183,7 +187,7 @@ WHERE mentioned_titles='{}'`)
 		if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
 			return nil, err
 		}
-		if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
+		if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
 			return []Review{}, err
 		}
 		reviews = append(reviews, r)
@@ -208,7 +212,7 @@ FROM review`)
 		if err := rows.Scan(&r.ID, &r.MovieID, &r.Source, &r.URL, &r.Review, &r.MovieRating, &r.Quality, &titles); err != nil {
 			return nil, err
 		}
-		if err := json.Unmarshal([]byte(titles), &r.Titles); err != nil {
+		if err := json.Unmarshal([]byte(titles), &r.Mentions); err != nil {
 			return []Review{}, err
 		}
 		reviews = append(reviews, r)

worker-client/worker/findalltitles.go

@@ -1,8 +1,6 @@
 package worker
 
 import (
-	"time"
-
 	"code.ewintr.nl/emdb/job"
 )
 
@@ -17,7 +15,6 @@ func (w *Worker) FindAllTitles(jobID int) {
 	}
 
 	for _, r := range reviews {
-		time.Sleep(1 * time.Second)
 		if err := w.jq.Add(r.ID, job.ActionFindTitles); err != nil {
 			logger.Error("could not add job", "error", err)
 			w.jq.MarkFailed(jobID)

worker-client/worker/findtitles.go

@@ -3,6 +3,8 @@ package worker
 import (
 	"encoding/json"
 	"fmt"
+
+	"code.ewintr.nl/emdb/storage"
 )
 
 const (
@@ -12,11 +14,13 @@ const (
 %s
 ---- 
 
-If you found any movie titles other than %s, list them below in a JSON array. If there are other titles, like TV shows, books or games, ignore them. The format is as follows:
+If you found any movie titles other than %s, list them below in a JSON array. If there are titles of other media, like TV shows, books or games, ignore them. The format is as follows:
 
-["movie title 1", "movie title 2"]
+---
+{"titles": ["movie title 1", "movie title 2"]}
+---
 
-Just answer with the JSON and nothing else. If you don't see any other movie titles, just answer with an empty JSON array.`
+Just answer with the JSON and nothing else. If you don't see any other movie titles, just use an empty JSON array.`
 )
 
 func (w *Worker) FindTitles(jobID int, reviewID string) {
@@ -42,23 +46,26 @@ func (w *Worker) FindTitles(jobID int, reviewID string) {
 	}
 
 	prompt := fmt.Sprintf(mentionsTemplate, movieTitle, review.Review, movieTitle)
-	titles, err := w.ollama.Generate("mistral", prompt)
+	resp, err := w.ollama.Generate("mistral", prompt)
 	if err != nil {
 		logger.Error("could not find titles: %w", err)
 	}
-	logger.Info("checked review", "found", titles)
+	logger.Info("checked review", "found", resp)
-	var resp []string
+	var mentions storage.TitleMentions
-	if err := json.Unmarshal([]byte(titles), &resp); err != nil {
+	if err := json.Unmarshal([]byte(resp), &mentions); err != nil {
 		logger.Error("could not unmarshal llm response", "error", err)
 		w.jq.MarkFailed(jobID)
 		return
 	}
 
-	review.Titles = resp
+	review.Mentions = mentions
 
 	if err := w.reviewRepo.Store(review); err != nil {
 		logger.Error("could not update review", "error", err)
 		w.jq.MarkFailed(jobID)
 		return
 	}
+
+	logger.Info("done finding title mentions", "count", len(mentions.Titles))
+	w.jq.MarkDone(jobID)
 }

worker-client/worker/worker.go

@@ -45,7 +45,7 @@ func (w *Worker) Run() {
 		j, err := w.jq.Next()
 		switch {
 		case errors.Is(err, sql.ErrNoRows):
-			logger.Info("no jobs found")
+			//logger.Info("no jobs found")
 			continue
 		case err != nil:
 			logger.Error("could not get next job", "error", err)