ewintr
/
yogai
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

multiple pipelines

This commit is contained in:
Erik Winter 2023-07-06 13:37:55 +02:00
parent 5d125c0ac1
commit f4c856df68
2 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
process/processor.go
View File

@ -65,6 +65,10 @@ func (p *Pipeline) Process(ctx context.Context, video *model.Video) {
next := p.procs.Next(video)
if next == nil {
p.logger.Info("no more processors for video", slog.String("video", string(video.YoutubeID)))
video.Status = model.StatusReady
if err := p.relStorage.Save(video); err != nil {
p.logger.Error("failed to save video in rel db", slog.String("video", string(video.YoutubeID)), slog.String("error", err.Error()))
}
return
}

5
service.go
View File

@ -78,8 +78,9 @@ func main() {
logger.Info("fetch service started")
procs := process.NewProcessors(openAIClient)
pipeline := process.NewPipeline(fetcher.Out(), procs, videoRelRepo, wvClient, logger)
go pipeline.Run()
for i := 0; i < 4; i++ {
go process.NewPipeline(fetcher.Out(), procs, videoRelRepo, wvClient, logger.With(slog.Int("pipeline", i))).Run()
}
logger.Info("processing service started")
port, err := strconv.Atoi(getParam("API_PORT", "8080"))