Stabilize search pipeline and improve preview diagnostics

backend/handlers/api.go

@@ -76,6 +76,27 @@ type PreviewResponse struct {
 	Qualities        []map[string]any `json:"qualities"`
 }
 
+type searchDebugSummary struct {
+	Total              int              `json:"total"`
+	BySource           map[string]int   `json:"bySource"`
+	WithPreview        int              `json:"withPreview"`
+	WithThumbnail      int              `json:"withThumbnail"`
+	Top                []map[string]any `json:"top"`
+	Warning            string           `json:"warning,omitempty"`
+	DurationMS         int64            `json:"durationMs,omitempty"`
+	GeminiCandidateCap int              `json:"geminiCandidateCap,omitempty"`
+}
+
+type geminiBatchStats struct {
+	CandidateCap     int      `json:"candidateCap"`
+	Requested        int      `json:"requested"`
+	Batches          int      `json:"batches"`
+	Succeeded        int      `json:"succeeded"`
+	Failed           int      `json:"failed"`
+	RecommendedCount int      `json:"recommendedCount"`
+	Errors           []string `json:"errors,omitempty"`
+}
+
 func RegisterRoutes(router *gin.Engine, app *App) {
 	router.GET("/healthz", func(c *gin.Context) {
 		c.JSON(http.StatusOK, gin.H{"status": "ok"})
@@ -259,6 +280,7 @@ func (a *App) runDownload(recordID int64, url, start, end, quality, outputPath s
 }
 
 func (a *App) searchMedia(c *gin.Context) {
+	started := time.Now()
 	var req struct {
 		Query     string   `json:"query"`
 		Platforms []string `json:"platforms"`
@@ -277,18 +299,24 @@ func (a *App) searchMedia(c *gin.Context) {
 	if len(queryVariants) == 0 {
 		queryVariants = []string{req.Query}
 	}
-	a.debug("search query variants", gin.H{"query": req.Query, "variants": queryVariants, "platforms": req.Platforms})
+	a.debug("search query variants", gin.H{
+		"query":         req.Query,
+		"platforms":     req.Platforms,
+		"variants":      queryVariants,
+		"variantCount":  len(queryVariants),
+		"requestIdHint": time.Now().UnixNano(),
+	})
 
 	enabledPlatforms := normalizePlatforms(req.Platforms)
 	a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "searching " + selectedPlatformLabel(enabledPlatforms), "progress": 35})
 	results, err := a.SearchService.SearchMedia(queryVariants, enabledPlatforms)
 	if err != nil {
-		a.debug("search backend failed", gin.H{"error": err.Error(), "variants": queryVariants})
+		a.debug("search backend failed", gin.H{"error": err.Error(), "variants": queryVariants, "durationMs": time.Since(started).Milliseconds()})
 		a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "search failed", "progress": 100, "message": err.Error()})
 		c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
 		return
 	}
-	a.debug("search backend results", gin.H{"count": len(results), "results": results})
+	a.debug("search backend summary", summarizeSearchResults(results, time.Since(started), 0, ""))
 	if len(results) == 0 {
 		warning := "SearXNG returned no renderable results."
 		a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "no renderable search results", "progress": 100, "message": warning})
@@ -302,10 +330,10 @@ func (a *App) searchMedia(c *gin.Context) {
 		rankQuery = strings.Join(queryVariants[:min(len(queryVariants), 3)], " ")
 	}
 	scored := rankSearchResults(rankQuery, results)
-	a.debug("search ranked results", gin.H{"count": len(scored), "results": scored})
-	a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "analyzing all candidate visuals with Gemini Vision", "progress": 75})
-	recommended := evaluateAllCandidatesWithGemini(a.GeminiService, req.Query, scored)
-	a.debug("search gemini recommendations", gin.H{"count": len(recommended), "results": recommended})
+	a.debug("search ranked summary", summarizeSearchResults(scored, time.Since(started), geminiCandidateLimit(len(scored)), ""))
+	a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "analyzing top candidate visuals with Gemini Vision", "progress": 75})
+	recommended, geminiStats := evaluateAllCandidatesWithGemini(a.GeminiService, req.Query, scored)
+	a.debug("search gemini evaluation", geminiStats)
 	err = nil
 	if len(recommended) == 0 {
 		err = fmt.Errorf("gemini vision returned no recommended items across all candidate batches")
@@ -316,6 +344,7 @@ func (a *App) searchMedia(c *gin.Context) {
 			fallback = append(fallback, services.AIRecommendation{
 				Title:           result.Title,
 				Link:            result.Link,
+				Snippet:         result.Snippet,
 				ThumbnailURL:    result.ThumbnailURL,
 				PreviewVideoURL: result.PreviewVideoURL,
 				Source:          result.Source,
@@ -324,12 +353,15 @@ func (a *App) searchMedia(c *gin.Context) {
 			})
 		}
 		warning := err.Error()
+		a.debug("search fallback summary", summarizeRecommendationResults(fallback, time.Since(started), warning))
 		a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "Gemini Vision fallback to ranked results", "progress": 90, "message": warning})
 		c.JSON(http.StatusOK, gin.H{"results": fallback, "warning": warning, "queries": queryVariants})
 		return
 	}
 
-	response := gin.H{"results": mergeRecommendations(recommended, scored, 20), "queries": queryVariants}
+	merged := mergeRecommendations(recommended, scored, 20)
+	a.debug("search complete summary", summarizeRecommendationResults(merged, time.Since(started), ""))
+	response := gin.H{"results": merged, "queries": queryVariants}
 	a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "search complete", "progress": 100})
 	c.JSON(http.StatusOK, response)
 }
@@ -406,20 +438,31 @@ func selectedPlatformLabel(platforms map[string]bool) string {
 	return strings.Join(labels, ", ")
 }
 
-func evaluateAllCandidatesWithGemini(service *services.GeminiService, query string, ranked []services.SearchResult) []services.AIRecommendation {
+func evaluateAllCandidatesWithGemini(service *services.GeminiService, query string, ranked []services.SearchResult) ([]services.AIRecommendation, geminiBatchStats) {
 	const chunkSize = 8
+	limit := geminiCandidateLimit(len(ranked))
+	stats := geminiBatchStats{
+		CandidateCap: limit,
+		Requested:    min(limit, len(ranked)),
+	}
 	merged := make([]services.AIRecommendation, 0, len(ranked))
 	seen := map[string]bool{}
-	for start := 0; start < len(ranked); start += chunkSize {
+	for start := 0; start < limit; start += chunkSize {
 		end := start + chunkSize
-		if end > len(ranked) {
-			end = len(ranked)
+		if end > limit {
+			end = limit
 		}
 		batch := ranked[start:end]
+		stats.Batches++
 		recommended, err := service.Recommend(query, batch)
 		if err != nil {
+			stats.Failed++
+			if len(stats.Errors) < 5 {
+				stats.Errors = append(stats.Errors, err.Error())
+			}
 			continue
 		}
+		stats.Succeeded++
 		for _, item := range recommended {
 			if item.Link == "" || seen[item.Link] {
 				continue
@@ -428,7 +471,8 @@ func evaluateAllCandidatesWithGemini(service *services.GeminiService, query stri
 			merged = append(merged, item)
 		}
 	}
-	return merged
+	stats.RecommendedCount = len(merged)
+	return merged, stats
 }
 
 func rankSearchResults(query string, results []services.SearchResult) []services.SearchResult {
@@ -515,6 +559,7 @@ func mergeRecommendations(recommended []services.AIRecommendation, ranked []serv
 		merged = append(merged, services.AIRecommendation{
 			Title:           item.Title,
 			Link:            item.Link,
+			Snippet:         item.Snippet,
 			ThumbnailURL:    item.ThumbnailURL,
 			PreviewVideoURL: item.PreviewVideoURL,
 			Source:          item.Source,
@@ -525,6 +570,96 @@ func mergeRecommendations(recommended []services.AIRecommendation, ranked []serv
 	return merged
 }
 
+func geminiCandidateLimit(total int) int {
+	switch {
+	case total <= 8:
+		return total
+	case total <= 16:
+		return 12
+	default:
+		return 16
+	}
+}
+
+func summarizeSearchResults(results []services.SearchResult, duration time.Duration, geminiCap int, warning string) searchDebugSummary {
+	bySource := map[string]int{}
+	withPreview := 0
+	withThumbnail := 0
+	top := make([]map[string]any, 0, min(6, len(results)))
+	for idx, item := range results {
+		bySource[item.Source]++
+		if strings.TrimSpace(item.PreviewVideoURL) != "" {
+			withPreview++
+		}
+		if strings.TrimSpace(item.ThumbnailURL) != "" {
+			withThumbnail++
+		}
+		if idx < 6 {
+			top = append(top, map[string]any{
+				"title":         truncateText(item.Title, 120),
+				"source":        item.Source,
+				"hasPreview":    item.PreviewVideoURL != "",
+				"hasThumbnail":  item.ThumbnailURL != "",
+				"displayLink":   item.DisplayLink,
+				"snippetSample": truncateText(item.Snippet, 160),
+			})
+		}
+	}
+	return searchDebugSummary{
+		Total:              len(results),
+		BySource:           bySource,
+		WithPreview:        withPreview,
+		WithThumbnail:      withThumbnail,
+		Top:                top,
+		Warning:            warning,
+		DurationMS:         duration.Milliseconds(),
+		GeminiCandidateCap: geminiCap,
+	}
+}
+
+func summarizeRecommendationResults(results []services.AIRecommendation, duration time.Duration, warning string) searchDebugSummary {
+	bySource := map[string]int{}
+	withPreview := 0
+	withThumbnail := 0
+	top := make([]map[string]any, 0, min(6, len(results)))
+	for idx, item := range results {
+		bySource[item.Source]++
+		if strings.TrimSpace(item.PreviewVideoURL) != "" {
+			withPreview++
+		}
+		if strings.TrimSpace(item.ThumbnailURL) != "" {
+			withThumbnail++
+		}
+		if idx < 6 {
+			top = append(top, map[string]any{
+				"title":         truncateText(item.Title, 120),
+				"source":        item.Source,
+				"hasPreview":    item.PreviewVideoURL != "",
+				"hasThumbnail":  item.ThumbnailURL != "",
+				"reasonSample":  truncateText(item.Reason, 120),
+				"snippetSample": truncateText(item.Snippet, 160),
+			})
+		}
+	}
+	return searchDebugSummary{
+		Total:         len(results),
+		BySource:      bySource,
+		WithPreview:   withPreview,
+		WithThumbnail: withThumbnail,
+		Top:           top,
+		Warning:       warning,
+		DurationMS:    duration.Milliseconds(),
+	}
+}
+
+func truncateText(text string, limit int) string {
+	trimmed := strings.TrimSpace(text)
+	if len(trimmed) <= limit {
+		return trimmed
+	}
+	return trimmed[:limit] + "..."
+}
+
 func EnsurePaths(downloadsDir, workerScript string) error {
 	if err := os.MkdirAll(downloadsDir, 0o755); err != nil {
 		return err