Optimize backend search and evaluation pipeline

backend/services/ranker.go

@@ -260,6 +260,72 @@ func RandomizeTopRecommendations(items []AIRecommendation, window int) []AIRecom
 	return shuffled
 }
 
+func ReviewedRecommendationLinks(items []AIRecommendation) map[string]bool {
+	seen := map[string]bool{}
+	for _, item := range items {
+		if item.Link == "" {
+			continue
+		}
+		seen[item.Link] = true
+	}
+	return seen
+}
+
+func RemainingGeminiCapacity(reviewed []AIRecommendation) int {
+	remaining := GeminiCandidateLimit(16) - len(ReviewedRecommendationLinks(reviewed))
+	if remaining < 0 {
+		return 0
+	}
+	return remaining
+}
+
+func SelectUnevaluatedCandidates(ranked []SearchResult, reviewedLinks map[string]bool, limit int) []SearchResult {
+	if limit <= 0 {
+		return []SearchResult{}
+	}
+
+	selected := make([]SearchResult, 0, min(limit, len(ranked)))
+	for _, item := range ranked {
+		if len(selected) >= limit {
+			break
+		}
+		if item.Link == "" || reviewedLinks[item.Link] {
+			continue
+		}
+		selected = append(selected, item)
+	}
+	return selected
+}
+
+func MergeUniqueRecommendations(base, extra []AIRecommendation) []AIRecommendation {
+	merged := make([]AIRecommendation, 0, len(base)+len(extra))
+	seen := map[string]bool{}
+	for _, item := range append(base, extra...) {
+		if item.Link == "" || seen[item.Link] {
+			continue
+		}
+		seen[item.Link] = true
+		merged = append(merged, item)
+	}
+	return merged
+}
+
+func MergeGeminiBatchStats(base, extra GeminiBatchStats) GeminiBatchStats {
+	merged := base
+	merged.CandidateCap += extra.CandidateCap
+	merged.Requested += extra.Requested
+	merged.Batches += extra.Batches
+	merged.Succeeded += extra.Succeeded
+	merged.Failed += extra.Failed
+	merged.SequentialRetried += extra.SequentialRetried
+	merged.RecommendedCount += extra.RecommendedCount
+	merged.Errors = append(append([]string{}, base.Errors...), extra.Errors...)
+	if len(merged.Errors) > 5 {
+		merged.Errors = merged.Errors[:5]
+	}
+	return merged
+}
+
 func recoverGeminiBatchSequentially(service *GeminiService, query string, ranked []SearchResult, startIndex int) ([]AIRecommendation, []string) {
 	recovered := make([]AIRecommendation, 0, 8)
 	errs := make([]string, 0, 4)
@@ -270,11 +336,11 @@ func recoverGeminiBatchSequentially(service *GeminiService, query string, ranked
 			if len(errs) < 4 {
 				errs = append(errs, err.Error())
 			}
-			time.Sleep(350 * time.Millisecond)
+			time.Sleep(150 * time.Millisecond)
 			continue
 		}
 		recovered = append(recovered, recs...)
-		time.Sleep(350 * time.Millisecond)
+		time.Sleep(150 * time.Millisecond)
 	}
 	return recovered, errs
 }