From 91ee37593c90a9caf78f5b64a95e807eb91b8c53 Mon Sep 17 00:00:00 2001 From: GHStaK Date: Tue, 17 Mar 2026 16:06:59 +0900 Subject: [PATCH] Fix gemini candidate starvation --- TODO.md | 24 ++++++++++++++++++++++++ backend/services/cse.go | 23 +++++++++++++++++++---- backend/services/cse_test.go | 24 ++++++++++++++++++++++++ frontend/app.js | 6 +++--- 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/TODO.md b/TODO.md index ec8ea0a..cfa26e8 100644 --- a/TODO.md +++ b/TODO.md @@ -655,6 +655,30 @@ - If behavior in the browser does not match the latest backend/frontend code, the first assumption should be stale frontend assets until proven otherwise ## Recent Change Log +- Date: `2026-03-17` +- What changed: + - Fixed a search-budget regression where source collection could consume the full `SearchService` deadline and leave no time for Envato / Artgrid enrichment, causing Gemini to see only missing or low-value visuals. + - Split the search-service deadline into: + - collector deadline + - enrichment deadline with an explicit reserved window + - Added unit coverage for the new deadline split behavior. + - Stopped frontend preview-probe fallback from calling `/api/download/preview` for Artgrid items that do not already have a provider preview URL, so unsupported `yt-dlp` Artgrid probe errors no longer fire just from opening or hovering those results. +- Why it changed: + - The user-provided log `ai-media-hub-2026-03-17T07-01-21-282Z.log` showed: + - `search_service:deadline_reached` + - immediate `search_service:enrich_start` -> `search_service:enrich_complete` + - `withPreview: 0` + - `withLowValueThumbnail: 12` + - repeated `candidate has no thumbnail or preview video` + - final warning `gemini vision returned no candidate evaluations` + - The same log also showed Artgrid preview probe failures from `yt-dlp` returning `Unsupported URL`, which were not helping user-facing preview behavior. +- How it was verified: + - `pwsh -NoProfile -File scripts/selftest.ps1` + - added Go tests for the search/enrichment deadline split helper +- What is still risky or incomplete: + - This preserves time for enrichment, but it does not guarantee that every live Envato / Artgrid page yields a usable preview URL. + - Artgrid still depends on backend-enriched provider preview URLs for true video preview; if no provider preview is discovered, the UI will still fall back to thumbnail-only rendering. + - Date: `2026-03-17` - What changed: - Added repo-local Windows 11 PowerShell workflows: diff --git a/backend/services/cse.go b/backend/services/cse.go index f995bce..5775af3 100644 --- a/backend/services/cse.go +++ b/backend/services/cse.go @@ -53,6 +53,8 @@ type SearchExecutionMeta struct { PartialDueToDeadline bool `json:"partialDueToDeadline"` } +const searchEnrichmentReserve = 4 * time.Second + func NewSearchService(baseURL, googleVideoEngine, webEngine string) *SearchService { if googleVideoEngine == "" { googleVideoEngine = "google videos" @@ -84,9 +86,11 @@ func (s *SearchService) SearchMediaWithDeadline(queries []string, enabledPlatfor if s.BaseURL == "" { return nil, meta, fmt.Errorf("searxng base url is not configured") } + collectionDeadline, enrichmentDeadline := splitSearchDeadlines(deadline) s.debug("search_service:start", map[string]any{ "queries": queries, "enabledPlatforms": enabledPlatforms, + "deadlineSet": !deadline.IsZero(), }) seen := map[string]bool{} @@ -99,7 +103,7 @@ func (s *SearchService) SearchMediaWithDeadline(queries []string, enabledPlatfor primaryQueries := baseQueries[:minInt(len(baseQueries), 4)] runSearchPass := func(bases []string, onlyMissing bool) { for _, base := range bases { - if !deadline.IsZero() && time.Now().After(deadline) { + if !collectionDeadline.IsZero() && time.Now().After(collectionDeadline) { meta.PartialDueToDeadline = true s.debug("search_service:deadline_reached", map[string]any{"stage": "runSearchPass", "base": base}) return @@ -109,7 +113,7 @@ func (s *SearchService) SearchMediaWithDeadline(queries []string, enabledPlatfor continue } for _, collector := range s.collectors { - if !deadline.IsZero() && time.Now().After(deadline) { + if !collectionDeadline.IsZero() && time.Now().After(collectionDeadline) { meta.PartialDueToDeadline = true s.debug("search_service:deadline_reached", map[string]any{"stage": "collectorLoop", "collector": collector.Name()}) return @@ -133,7 +137,7 @@ func (s *SearchService) SearchMediaWithDeadline(queries []string, enabledPlatfor "searchQueries": searchQueries, }) for _, searchQuery := range searchQueries { - if !deadline.IsZero() && time.Now().After(deadline) { + if !collectionDeadline.IsZero() && time.Now().After(collectionDeadline) { meta.PartialDueToDeadline = true s.debug("search_service:deadline_reached", map[string]any{"stage": "queryLoop", "collector": collector.Name(), "query": searchQuery}) return @@ -192,11 +196,22 @@ func (s *SearchService) SearchMediaWithDeadline(queries []string, enabledPlatfor "hadError": lastErr != nil, "partialDueToDeadline": meta.PartialDueToDeadline, }) - enriched, enrichMeta := s.EnrichResultsWithDeadline(results, deadline) + enriched, enrichMeta := s.EnrichResultsWithDeadline(results, enrichmentDeadline) meta.PartialDueToDeadline = meta.PartialDueToDeadline || enrichMeta.PartialDueToDeadline return enriched, meta, nil } +func splitSearchDeadlines(deadline time.Time) (time.Time, time.Time) { + if deadline.IsZero() { + return time.Time{}, time.Time{} + } + remaining := time.Until(deadline) + if remaining <= searchEnrichmentReserve { + return deadline, deadline + } + return deadline.Add(-searchEnrichmentReserve), deadline +} + func (s *SearchService) EnrichResults(results []SearchResult) []SearchResult { enriched, _ := s.EnrichResultsWithDeadline(results, time.Time{}) return enriched diff --git a/backend/services/cse_test.go b/backend/services/cse_test.go index 397790e..eec9f8b 100644 --- a/backend/services/cse_test.go +++ b/backend/services/cse_test.go @@ -182,6 +182,30 @@ func TestSearchServiceFetchCacheRoundTrip(t *testing.T) { } } +func TestSplitSearchDeadlinesReservesEnrichmentWindow(t *testing.T) { + deadline := time.Now().Add(20 * time.Second) + collectionDeadline, enrichmentDeadline := splitSearchDeadlines(deadline) + + if enrichmentDeadline.IsZero() { + t.Fatal("expected enrichment deadline to be preserved") + } + if !collectionDeadline.Before(enrichmentDeadline) { + t.Fatalf("expected collection deadline before enrichment deadline, got %v >= %v", collectionDeadline, enrichmentDeadline) + } + if gap := enrichmentDeadline.Sub(collectionDeadline); gap < searchEnrichmentReserve-500*time.Millisecond { + t.Fatalf("expected reserve close to %v, got %v", searchEnrichmentReserve, gap) + } +} + +func TestSplitSearchDeadlinesDoesNotReserveWhenDeadlineIsTooClose(t *testing.T) { + deadline := time.Now().Add(2 * time.Second) + collectionDeadline, enrichmentDeadline := splitSearchDeadlines(deadline) + + if !collectionDeadline.Equal(enrichmentDeadline) { + t.Fatalf("expected identical deadlines when budget is too tight, got %v and %v", collectionDeadline, enrichmentDeadline) + } +} + func TestSearchServiceSkipsArtgridAPIAfter403(t *testing.T) { var apiRequests atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/frontend/app.js b/frontend/app.js index 3932336..2296456 100644 --- a/frontend/app.js +++ b/frontend/app.js @@ -691,10 +691,10 @@ function renderResults(results) { node.addEventListener("click", () => openResultModal(item)); previewVideo.poster = usableThumbnail ? item.thumbnailUrl : ""; const mediaArea = node.querySelector(".relative"); - if (item.previewVideoUrl || item.source === "Google Video" || item.source === "Artgrid") { + if (item.previewVideoUrl || item.source === "Google Video") { mediaArea.addEventListener("mouseenter", async () => { let previewURL = item.previewVideoUrl || ""; - if (!previewURL && (item.source === "Google Video" || item.source === "Artgrid")) { + if (!previewURL && item.source === "Google Video") { const preview = await fetchResultPreview(item); previewURL = preview?.previewStreamUrl || ""; } @@ -769,7 +769,7 @@ async function openResultModal(item) { const embedURL = buildResultModalEmbedURL(item); const fallbackReason = item.previewBlockedReason || "Embedded view was unavailable, switched to fallback preview."; let resolvedPreviewURL = item.previewVideoUrl || ""; - if (!resolvedPreviewURL && (item.source === "Google Video" || item.source === "Artgrid")) { + if (!resolvedPreviewURL && item.source === "Google Video") { const preview = await fetchResultPreview(item); resolvedPreviewURL = preview?.previewStreamUrl || ""; }