Return partial search results before proxy timeout

TODO.md

@@ -499,3 +499,15 @@
   - `go test ./...`
 - What is still risky or incomplete:
   - If the browser is holding an older cached `app.js`, a hard refresh may still be needed before the proxied preview path is actually used on the client.
+
+- Date: `2026-03-16`
+- What changed:
+  - Added a request-level time budget so search collection, enrichment, supplemental exploration, and Gemini evaluation stop early enough to return partial results before the reverse proxy reaches `504 Gateway Time-out`.
+  - Reduced query variant count and per-source caps slightly again to cut the worst-case number of SearXNG calls in a single search request.
+- Why it changed:
+  - The new debug log showed `/api/search` taking about `90s`, with the timeout happening during the Gemini stage after a very large number of search and enrichment steps.
+- How it was verified:
+  - `go test ./...`
+  - `bash scripts/selftest.sh`
+- What is still risky or incomplete:
+  - Partial-result responses are now preferred over hard 504s, so some searches may return fewer reviewed items when the time budget is exhausted.

backend/handlers/api.go

@@ -383,6 +383,7 @@ func (a *App) runDownload(recordID int64, url, start, end, quality, outputPath s
 
 func (a *App) searchMedia(c *gin.Context) {
 	started := time.Now()
+	deadline := started.Add(45 * time.Second)
 	var req struct {
 		Query     string   `json:"query"`
 		Platforms []string `json:"platforms"`
@@ -411,7 +412,7 @@ func (a *App) searchMedia(c *gin.Context) {
 
 	enabledPlatforms := normalizePlatforms(req.Platforms)
 	a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "searching " + selectedPlatformLabel(enabledPlatforms), "progress": 35})
-	results, err := a.SearchService.SearchMedia(queryVariants, enabledPlatforms)
+	results, err := a.SearchService.SearchMediaWithDeadline(queryVariants, enabledPlatforms, deadline.Add(-20*time.Second))
 	if err != nil {
 		a.debug("search backend failed", gin.H{"error": err.Error(), "variants": queryVariants, "durationMs": time.Since(started).Milliseconds()})
 		a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "search failed", "progress": 100, "message": err.Error()})
@@ -434,16 +435,16 @@ func (a *App) searchMedia(c *gin.Context) {
 	scored := services.RankSearchResults(rankQuery, results)
 	a.debug("search ranked summary", summarizeSearchResults(scored, time.Since(started), services.GeminiCandidateLimit(len(scored)), ""))
 	a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "analyzing all candidate visuals with Gemini Vision", "progress": 75})
-	recommended, geminiStats, geminiErr := services.EvaluateAllCandidatesWithGemini(a.GeminiService, req.Query, scored)
+	recommended, geminiStats, geminiErr := services.EvaluateAllCandidatesWithGeminiWithDeadline(a.GeminiService, req.Query, scored, deadline.Add(-5*time.Second))
 	a.debug("search gemini evaluation", geminiStats)
-	if services.NeedsSupplementalExploration(recommended) {
+	if services.NeedsSupplementalExploration(recommended) && time.Now().Before(deadline.Add(-10*time.Second)) {
 		a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "Gemini 평가가 약해 추가 후보를 탐색하는 중", "progress": 82})
 		explorationQueries := buildSupplementalQueries(req.Query, queryVariants)
-		extraResults, extraErr := a.SearchService.SearchMedia(explorationQueries, enabledPlatforms)
+		extraResults, extraErr := a.SearchService.SearchMediaWithDeadline(explorationQueries, enabledPlatforms, deadline.Add(-10*time.Second))
 		if extraErr == nil && len(extraResults) > 0 {
 			results = mergeSearchResults(results, extraResults)
 			scored = services.RankSearchResults(strings.Join(explorationQueries[:min(len(explorationQueries), 3)], " "), results)
-			recommended, geminiStats, geminiErr = services.EvaluateAllCandidatesWithGemini(a.GeminiService, req.Query, scored)
+			recommended, geminiStats, geminiErr = services.EvaluateAllCandidatesWithGeminiWithDeadline(a.GeminiService, req.Query, scored, deadline.Add(-3*time.Second))
 			a.debug("search supplemental query variants", gin.H{"variants": explorationQueries, "variantCount": len(explorationQueries)})
 			a.debug("search gemini evaluation after supplemental search", geminiStats)
 		}
@@ -476,6 +477,9 @@ func (a *App) searchMedia(c *gin.Context) {
 	if warning != "" {
 		response["warning"] = warning
 	}
+	if time.Now().After(deadline.Add(-2*time.Second)) && warning == "" {
+		response["warning"] = "search returned partial results to avoid gateway timeout"
+	}
 	a.Hub.Broadcast("progress", gin.H{"type": "search", "status": "search complete", "progress": 100})
 	c.JSON(http.StatusOK, response)
 }

backend/services/cse.go

@@ -56,6 +56,10 @@ func NewSearchService(baseURL, googleVideoEngine, webEngine string) *SearchServi
 }
 
 func (s *SearchService) SearchMedia(queries []string, enabledPlatforms map[string]bool) ([]SearchResult, error) {
+	return s.SearchMediaWithDeadline(queries, enabledPlatforms, time.Time{})
+}
+
+func (s *SearchService) SearchMediaWithDeadline(queries []string, enabledPlatforms map[string]bool, deadline time.Time) ([]SearchResult, error) {
 	if s.BaseURL == "" {
 		return nil, fmt.Errorf("searxng base url is not configured")
 	}
@@ -69,16 +73,24 @@ func (s *SearchService) SearchMedia(queries []string, enabledPlatforms map[strin
 	results := make([]SearchResult, 0, 90)
 	var lastErr error
 
-	baseQueries := limitQueries(queries, 8)
+	baseQueries := limitQueries(queries, 6)
 	shuffleStrings(baseQueries)
-	primaryQueries := baseQueries[:minInt(len(baseQueries), 4)]
+	primaryQueries := baseQueries[:minInt(len(baseQueries), 3)]
 	runSearchPass := func(bases []string, onlyMissing bool) {
 		for _, base := range bases {
+			if !deadline.IsZero() && time.Now().After(deadline) {
+				s.debug("search_service:deadline_reached", map[string]any{"stage": "runSearchPass", "base": base})
+				return
+			}
 			base = strings.TrimSpace(base)
 			if base == "" {
 				continue
 			}
 			for _, collector := range s.collectors {
+				if !deadline.IsZero() && time.Now().After(deadline) {
+					s.debug("search_service:deadline_reached", map[string]any{"stage": "collectorLoop", "collector": collector.Name()})
+					return
+				}
 				if !collector.Enabled(enabledPlatforms) {
 					continue
 				}
@@ -97,6 +109,10 @@ func (s *SearchService) SearchMedia(queries []string, enabledPlatforms map[strin
 					"searchQueries": searchQueries,
 				})
 				for _, searchQuery := range searchQueries {
+					if !deadline.IsZero() && time.Now().After(deadline) {
+						s.debug("search_service:deadline_reached", map[string]any{"stage": "queryLoop", "collector": collector.Name(), "query": searchQuery})
+						return
+					}
 					if sourceCounts[collector.Name()] >= collector.MaxResults() {
 						break
 					}
@@ -150,10 +166,14 @@ func (s *SearchService) SearchMedia(queries []string, enabledPlatforms map[strin
 		"sourceCounts": sourceCounts,
 		"hadError":     lastErr != nil,
 	})
-	return s.EnrichResults(results), nil
+	return s.EnrichResultsWithDeadline(results, deadline), nil
 }
 
 func (s *SearchService) EnrichResults(results []SearchResult) []SearchResult {
+	return s.EnrichResultsWithDeadline(results, time.Time{})
+}
+
+func (s *SearchService) EnrichResultsWithDeadline(results []SearchResult, deadline time.Time) []SearchResult {
 	limit := minInt(len(results), 14)
 	if limit == 0 {
 		return results
@@ -172,6 +192,9 @@ func (s *SearchService) EnrichResults(results []SearchResult) []SearchResult {
 		wg.Add(1)
 		go func(i int) {
 			defer wg.Done()
+			if !deadline.IsZero() && time.Now().After(deadline) {
+				return
+			}
 			sem <- struct{}{}
 			defer func() { <-sem }()
 			s.debug("search_service:enrich_item_start", map[string]any{

backend/services/ranker.go

@@ -91,6 +91,10 @@ func GeminiCandidateLimit(total int) int {
 }
 
 func EvaluateAllCandidatesWithGemini(service *GeminiService, query string, ranked []SearchResult) ([]AIRecommendation, GeminiBatchStats, error) {
+	return EvaluateAllCandidatesWithGeminiWithDeadline(service, query, ranked, time.Time{})
+}
+
+func EvaluateAllCandidatesWithGeminiWithDeadline(service *GeminiService, query string, ranked []SearchResult, deadline time.Time) ([]AIRecommendation, GeminiBatchStats, error) {
 	const chunkSize = 8
 	const maxConcurrentBatches = 2
 	if service == nil {
@@ -135,6 +139,13 @@ func EvaluateAllCandidatesWithGemini(service *GeminiService, query string, ranke
 		wg.Add(1)
 		go func(batchIndex int, candidates []SearchResult) {
 			defer wg.Done()
+			if !deadline.IsZero() && time.Now().After(deadline) {
+				results[batchIndex] = batchResult{
+					index: batchIndex,
+					err:   fmt.Errorf("skipped gemini batch due to deadline"),
+				}
+				return
+			}
 			sem <- struct{}{}
 			defer func() { <-sem }()
 			recommended, err := service.Recommend(query, candidates)

backend/services/search_collectors.go

@@ -15,7 +15,7 @@ type searchCollector interface {
 type envatoCollector struct{}
 
 func (envatoCollector) Name() string    { return "Envato" }
-func (envatoCollector) MaxResults() int { return 12 }
+func (envatoCollector) MaxResults() int { return 10 }
 func (envatoCollector) Enabled(enabledPlatforms map[string]bool) bool {
 	return len(enabledPlatforms) == 0 || enabledPlatforms["envato"]
 }
@@ -31,7 +31,7 @@ func (envatoCollector) Enrich(searcher *SearchService, result SearchResult) Sear
 type artgridCollector struct{}
 
 func (artgridCollector) Name() string    { return "Artgrid" }
-func (artgridCollector) MaxResults() int { return 12 }
+func (artgridCollector) MaxResults() int { return 10 }
 func (artgridCollector) Enabled(enabledPlatforms map[string]bool) bool {
 	return len(enabledPlatforms) == 0 || enabledPlatforms["artgrid"]
 }
@@ -47,7 +47,7 @@ func (artgridCollector) Enrich(searcher *SearchService, result SearchResult) Sea
 type googleVideoCollector struct{}
 
 func (googleVideoCollector) Name() string    { return "Google Video" }
-func (googleVideoCollector) MaxResults() int { return 8 }
+func (googleVideoCollector) MaxResults() int { return 6 }
 func (googleVideoCollector) Enabled(enabledPlatforms map[string]bool) bool {
 	return len(enabledPlatforms) == 0 || enabledPlatforms["google video"]
 }