Return partial search results before proxy timeout

backend/services/cse.go

@@ -56,6 +56,10 @@ func NewSearchService(baseURL, googleVideoEngine, webEngine string) *SearchServi
 }
 
 func (s *SearchService) SearchMedia(queries []string, enabledPlatforms map[string]bool) ([]SearchResult, error) {
+	return s.SearchMediaWithDeadline(queries, enabledPlatforms, time.Time{})
+}
+
+func (s *SearchService) SearchMediaWithDeadline(queries []string, enabledPlatforms map[string]bool, deadline time.Time) ([]SearchResult, error) {
 	if s.BaseURL == "" {
 		return nil, fmt.Errorf("searxng base url is not configured")
 	}
@@ -69,16 +73,24 @@ func (s *SearchService) SearchMedia(queries []string, enabledPlatforms map[strin
 	results := make([]SearchResult, 0, 90)
 	var lastErr error
 
-	baseQueries := limitQueries(queries, 8)
+	baseQueries := limitQueries(queries, 6)
 	shuffleStrings(baseQueries)
-	primaryQueries := baseQueries[:minInt(len(baseQueries), 4)]
+	primaryQueries := baseQueries[:minInt(len(baseQueries), 3)]
 	runSearchPass := func(bases []string, onlyMissing bool) {
 		for _, base := range bases {
+			if !deadline.IsZero() && time.Now().After(deadline) {
+				s.debug("search_service:deadline_reached", map[string]any{"stage": "runSearchPass", "base": base})
+				return
+			}
 			base = strings.TrimSpace(base)
 			if base == "" {
 				continue
 			}
 			for _, collector := range s.collectors {
+				if !deadline.IsZero() && time.Now().After(deadline) {
+					s.debug("search_service:deadline_reached", map[string]any{"stage": "collectorLoop", "collector": collector.Name()})
+					return
+				}
 				if !collector.Enabled(enabledPlatforms) {
 					continue
 				}
@@ -97,6 +109,10 @@ func (s *SearchService) SearchMedia(queries []string, enabledPlatforms map[strin
 					"searchQueries": searchQueries,
 				})
 				for _, searchQuery := range searchQueries {
+					if !deadline.IsZero() && time.Now().After(deadline) {
+						s.debug("search_service:deadline_reached", map[string]any{"stage": "queryLoop", "collector": collector.Name(), "query": searchQuery})
+						return
+					}
 					if sourceCounts[collector.Name()] >= collector.MaxResults() {
 						break
 					}
@@ -150,10 +166,14 @@ func (s *SearchService) SearchMedia(queries []string, enabledPlatforms map[strin
 		"sourceCounts": sourceCounts,
 		"hadError":     lastErr != nil,
 	})
-	return s.EnrichResults(results), nil
+	return s.EnrichResultsWithDeadline(results, deadline), nil
 }
 
 func (s *SearchService) EnrichResults(results []SearchResult) []SearchResult {
+	return s.EnrichResultsWithDeadline(results, time.Time{})
+}
+
+func (s *SearchService) EnrichResultsWithDeadline(results []SearchResult, deadline time.Time) []SearchResult {
 	limit := minInt(len(results), 14)
 	if limit == 0 {
 		return results
@@ -172,6 +192,9 @@ func (s *SearchService) EnrichResults(results []SearchResult) []SearchResult {
 		wg.Add(1)
 		go func(i int) {
 			defer wg.Done()
+			if !deadline.IsZero() && time.Now().After(deadline) {
+				return
+			}
 			sem <- struct{}{}
 			defer func() { <-sem }()
 			s.debug("search_service:enrich_item_start", map[string]any{