Add local self-test flow and fix fallback regressions

TODO.md

@@ -1,5 +1,43 @@
 # AI Media Hub Handover
 
+## Working Rule
+- From this point on, every meaningful change should be appended to this file so the next handoff can reconstruct:
+  - what changed
+  - why it changed
+  - how it was verified
+  - what remains risky
+- Treat this file as both backlog and handover log, not just a static TODO list.
+
+## Current Session Update (2026-03-13)
+- Added a local self-test workflow before push/container build:
+  - `scripts/selftest.sh`
+  - `scripts/mock_searxng.py`
+- Fixed Korean query translation fallback behavior:
+  - If `GEMINI_API_KEY` is missing or Gemini translation fails, the code now still attempts Google Translate fallback.
+  - If Google Translate fallback fails, dictionary replacement fallback still runs.
+- Added Go tests for translation fallback logic.
+- Fixed frontend HLS preview wiring:
+  - `hls.js` is now loaded in `frontend/index.html`
+  - frontend now tries `hls.js` first, then native HLS playback if available
+- Corrected the practical local verification note:
+  - `go build ./backend` from repo root conflicts with the existing `backend/` directory name
+  - verified build command is now treated as `go build -o /tmp/... ./backend`
+
+## Local Self-Test Workflow
+- Primary command:
+  - `bash scripts/selftest.sh`
+- What it currently verifies:
+  - Go formatting for touched backend files
+  - Python syntax for worker + mock SearXNG
+  - `go test ./...`
+  - backend binary build
+  - local app boot with temp SQLite/download dirs
+  - `/healthz`
+  - `/api/search` using a local mock SearXNG server
+  - `/api/upload`
+- Purpose:
+  - allow safe local regression checks before push or container build without depending on real SearXNG, Gemini, or browser interaction
+
 ## Project Summary
 - Project: `ai-media-hub`
 - Goal: AI-assisted media discovery + ingest dashboard for Unraid
@@ -106,6 +144,8 @@
 - Search relevance is still not considered stable enough.
 - Gemini batch evaluation exists, but search quality can still degrade if upstream SearXNG results are noisy.
 - Frontend JavaScript was not linted with Node tooling in this environment because `node` is not installed here.
+- Full browser-level preview validation is still not covered by the local self-test script.
+- Search cards still render recommendation reason text, not a robust asset description/snippet mapping.
 
 ## Frontend Debug Logger
 - UI button: bottom-right `Logs`
@@ -173,12 +213,17 @@
 - [ ] Envato / Artgrid preview extraction hardening
 - [ ] Search result relevance validation against real user queries
 - [ ] Better matching between rendered description and actual linked asset
+- [ ] Add browser-level verification for preview/HLS behavior
+- [ ] Add more automated coverage for search ranking / filtering logic
 - [ ] Add proper frontend build/lint step if Node becomes available
 
 ## Verified Locally In This Environment
-- [x] `go build ./backend`
+- [x] `go build -o /tmp/ai-media-hub ./backend`
-- [x] Python syntax check for worker at various stages
+- [x] `go test ./...` (currently no broad test suite beyond the added fallback tests)
-- [x] basic server boot / `/healthz` at various stages
+- [x] Python syntax check for worker + self-test helper
+- [x] local app boot / `/healthz` through `scripts/selftest.sh`
+- [x] local `/api/search` against mock SearXNG through `scripts/selftest.sh`
+- [x] local `/api/upload` through `scripts/selftest.sh`
 - [ ] full browser-level validation was not fully reproducible in this environment
 
 ## Short Handover Summary

backend/services/gemini.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 )
@@ -19,6 +20,8 @@ import (
 type GeminiService struct {
 	APIKey            string
 	Client            *http.Client
+	GenerateEndpoint  string
+	TranslateEndpoint string
 }
 
 type AIRecommendation struct {
@@ -39,6 +42,8 @@ func NewGeminiService(apiKey string) *GeminiService {
 	return &GeminiService{
 		APIKey:            apiKey,
 		Client:            &http.Client{Timeout: 40 * time.Second},
+		GenerateEndpoint:  "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent",
+		TranslateEndpoint: "https://translate.googleapis.com/translate_a/single",
 	}
 }
 
@@ -48,10 +53,15 @@ func (g *GeminiService) ExpandQuery(query string) ([]string, error) {
 }
 
 func (g *GeminiService) TranslateQuery(query string) string {
-	if strings.TrimSpace(query) == "" || looksMostlyASCII(query) || g.APIKey == "" {
+	trimmed := strings.TrimSpace(query)
-		return strings.TrimSpace(query)
+	if trimmed == "" {
+		return ""
+	}
+	if looksMostlyASCII(trimmed) {
+		return trimmed
 	}
 
+	if g.APIKey != "" {
 		body := map[string]any{
 			"systemInstruction": map[string]any{
 				"parts": []map[string]string{
@@ -64,7 +74,7 @@ func (g *GeminiService) TranslateQuery(query string) string {
 				{
 					"parts": []map[string]string{
 						{
-						"text": "Translate this user query into concise English suitable for stock-footage search: " + query,
+							"text": "Translate this user query into concise English suitable for stock-footage search: " + trimmed,
 						},
 					},
 				},
@@ -79,23 +89,24 @@ func (g *GeminiService) TranslateQuery(query string) string {
 		rawText, err := g.generateText(body)
 		if err == nil {
 			translated := sanitizePlainEnglishLine(rawText)
-		if translated != "" && !strings.EqualFold(translated, strings.TrimSpace(query)) {
+			if translated != "" && !strings.EqualFold(translated, trimmed) {
 				return translated
 			}
 		}
+	}
 
-	if translated, err := g.translateViaGoogle(query); err == nil && translated != "" {
+	if translated, err := g.translateViaGoogle(trimmed); err == nil && translated != "" && isLikelyEnglishQuery(translated) {
 		return translated
 	}
-	if translated := translateKoreanMediaTerms(query); translated != "" && !strings.EqualFold(translated, strings.TrimSpace(query)) {
+	if translated := translateKoreanMediaTerms(trimmed); translated != "" && !strings.EqualFold(translated, trimmed) {
 		return translated
 	}
-	return strings.TrimSpace(query)
+	return trimmed
 }
 
 func (g *GeminiService) generateText(body map[string]any) (string, error) {
 	rawBody, _ := json.Marshal(body)
-	endpoint := "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=" + g.APIKey
+	endpoint := strings.TrimRight(g.GenerateEndpoint, "?") + "?key=" + g.APIKey
 	resp, err := g.Client.Post(endpoint, "application/json", bytes.NewReader(rawBody))
 	if err != nil {
 		return "", fmt.Errorf("gemini request failed: %w", err)
@@ -168,7 +179,7 @@ User query: ` + query,
 	}
 
 	rawBody, _ := json.Marshal(body)
-	endpoint := "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=" + g.APIKey
+	endpoint := strings.TrimRight(g.GenerateEndpoint, "?") + "?key=" + g.APIKey
 	resp, err := g.Client.Post(endpoint, "application/json", bytes.NewReader(rawBody))
 	if err != nil {
 		return nil, err
@@ -448,41 +459,51 @@ func isLikelyEnglishQuery(text string) bool {
 }
 
 func translateKoreanMediaTerms(query string) string {
-	replacements := map[string]string{
+	replacements := []struct {
-		"숲속":  "forest",
+		korean  string
-		"숲":   "forest",
+		english string
-		"다정한": "affectionate",
+	}{
-		"커플":  "couple",
+		{korean: "숲속", english: "forest"},
-		"도시":  "city",
+		{korean: "다정한", english: "affectionate"},
-		"야경":  "night city",
+		{korean: "항공샷", english: "aerial shot"},
-		"거리":  "street",
+		{korean: "사람들", english: "people"},
-		"골목":  "alley",
+		{korean: "행복한", english: "happy"},
-		"바다":  "ocean",
+		{korean: "커플", english: "couple"},
-		"해변":  "beach",
+		{korean: "연인", english: "lovers"},
-		"노을":  "sunset",
+		{korean: "도시", english: "city"},
-		"자연":  "nature",
+		{korean: "야경", english: "night city"},
-		"비":   "rain",
+		{korean: "거리", english: "street"},
-		"눈":   "snow",
+		{korean: "골목", english: "alley"},
-		"드론":  "drone",
+		{korean: "바다", english: "ocean"},
-		"항공샷": "aerial shot",
+		{korean: "해변", english: "beach"},
-		"사람들": "people",
+		{korean: "노을", english: "sunset"},
-		"인파":  "crowd",
+		{korean: "자연", english: "nature"},
-		"행복한": "happy",
+		{korean: "드론", english: "drone"},
-		"연인":  "lovers",
+		{korean: "인파", english: "crowd"},
-		"공원":  "park",
+		{korean: "공원", english: "park"},
-		"산":   "mountain",
+		{korean: "숲", english: "forest"},
+		{korean: "비", english: "rain"},
+		{korean: "눈", english: "snow"},
+		{korean: "산", english: "mountain"},
 	}
+	sort.SliceStable(replacements, func(i, j int) bool {
+		return len([]rune(replacements[i].korean)) > len([]rune(replacements[j].korean))
+	})
 
 	translated := strings.TrimSpace(query)
-	for korean, english := range replacements {
+	for _, replacement := range replacements {
-		translated = strings.ReplaceAll(translated, korean, english)
+		translated = strings.ReplaceAll(translated, replacement.korean, replacement.english)
 	}
 	translated = strings.Join(strings.Fields(translated), " ")
 	return strings.TrimSpace(translated)
 }
 
 func (g *GeminiService) translateViaGoogle(query string) (string, error) {
-	endpoint := "https://translate.googleapis.com/translate_a/single?client=gtx&sl=auto&tl=en&dt=t&q=" + neturl.QueryEscape(query)
+	baseURL := g.TranslateEndpoint
+	if strings.TrimSpace(baseURL) == "" {
+		baseURL = "https://translate.googleapis.com/translate_a/single"
+	}
+	endpoint := baseURL + "?client=gtx&sl=auto&tl=en&dt=t&q=" + neturl.QueryEscape(query)
 	resp, err := g.Client.Get(endpoint)
 	if err != nil {
 		return "", err

backend/services/gemini_test.go

@@ -0,0 +1,41 @@
+package services
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestTranslateQueryFallsBackToGoogleWithoutGeminiKey(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`[[["rainy city street","비 오는 도시 거리",null,null,1]],null,"ko"]`))
+	}))
+	defer server.Close()
+
+	service := NewGeminiService("")
+	service.Client = &http.Client{Timeout: 2 * time.Second}
+	service.TranslateEndpoint = server.URL
+
+	translated := service.TranslateQuery("비 오는 도시 거리")
+	if translated != "rainy city street" {
+		t.Fatalf("expected google fallback translation, got %q", translated)
+	}
+}
+
+func TestTranslateQueryFallsBackToDictionaryWhenTranslateFails(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "boom", http.StatusBadGateway)
+	}))
+	defer server.Close()
+
+	service := NewGeminiService("")
+	service.Client = &http.Client{Timeout: 2 * time.Second}
+	service.TranslateEndpoint = server.URL
+
+	translated := service.TranslateQuery("숲속 커플")
+	if translated != "forest couple" {
+		t.Fatalf("expected dictionary fallback translation, got %q", translated)
+	}
+}

frontend/app.js

@@ -216,12 +216,21 @@ function attachVideoSource(video, src) {
     logEvent("preview:attach:skipped", { reason: "empty src" });
     return;
   }
-  if (src.endsWith(".m3u8") && window.Hls && window.Hls.isSupported()) {
+  if (src.endsWith(".m3u8")) {
+    if (window.Hls && window.Hls.isSupported()) {
       const hls = new window.Hls({ enableWorker: false });
       hls.loadSource(src);
       hls.attachMedia(video);
       hlsInstances.set(video, hls);
-    logEvent("preview:attach:hls", { src });
+      logEvent("preview:attach:hls", { src, mode: "hls.js" });
+      return;
+    }
+    if (video.canPlayType("application/vnd.apple.mpegurl")) {
+      video.src = src;
+      logEvent("preview:attach:hls", { src, mode: "native" });
+      return;
+    }
+    logEvent("preview:attach:skipped", { reason: "hls unsupported", src });
     return;
   }
   video.src = src;

frontend/index.html

@@ -5,6 +5,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>AI Media Hub</title>
     <script src="https://cdn.tailwindcss.com"></script>
+    <script src="https://cdn.jsdelivr.net/npm/hls.js@1.5.20/dist/hls.min.js"></script>
     <link rel="stylesheet" href="/style.css" />
   </head>
   <body class="min-h-full bg-zinc-950 text-zinc-100 selection:bg-white selection:text-black">

scripts/mock_searxng.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+import argparse
+import json
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import parse_qs, urlparse
+
+
+def build_results(engine: str, query: str):
+    lowered = (engine or "").lower()
+    if "google" in lowered and "video" in lowered:
+        return [
+            {
+                "title": f"{query.title()} cinematic b-roll",
+                "url": "https://www.youtube.com/watch?v=abcdefghijk",
+                "content": f"{query} stock footage edit reference",
+                "thumbnail": "https://i.ytimg.com/vi/abcdefghijk/hqdefault.jpg",
+                "engine": engine,
+            }
+        ]
+
+    return [
+        {
+            "title": f"{query.title()} envato clip",
+            "url": "https://elements.envato.com/city-rain-b-roll-AB12CD",
+            "content": f"{query} stock footage cinematic scene",
+            "thumbnail": "https://images.example.com/envato-city-rain.jpg",
+            "thumbnail_src": "https://images.example.com/envato-city-rain.jpg",
+            "engine": engine,
+        },
+        {
+            "title": f"{query.title()} artgrid clip",
+            "url": "https://artgrid.io/clip/123456/city-rain-night",
+            "content": f"{query} editorial footage establishing shot",
+            "thumbnail": "https://images.example.com/artgrid-city-rain.jpg",
+            "thumbnail_src": "https://images.example.com/artgrid-city-rain.jpg",
+            "img_src": "https://images.example.com/artgrid-city-rain.jpg",
+            "engine": engine,
+        },
+    ]
+
+
+class Handler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        parsed = urlparse(self.path)
+        if parsed.path != "/search":
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        params = parse_qs(parsed.query)
+        query = params.get("q", ["city rain"])[0]
+        engine = params.get("engines", [""])[0]
+        payload = {"results": build_results(engine, query)}
+
+        body = json.dumps(payload).encode("utf-8")
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, format, *args):
+        return
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=18080)
+    args = parser.parse_args()
+
+    server = ThreadingHTTPServer(("127.0.0.1", args.port), Handler)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        server.server_close()
+
+
+if __name__ == "__main__":
+    main()

scripts/selftest.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+TMP_DIR="$(mktemp -d)"
+MOCK_PORT="${SELFTEST_MOCK_PORT:-18080}"
+APP_PORT="${SELFTEST_APP_PORT:-18081}"
+MOCK_PID=""
+APP_PID=""
+
+cleanup() {
+  if [[ -n "${APP_PID}" ]] && kill -0 "${APP_PID}" >/dev/null 2>&1; then
+    kill "${APP_PID}" >/dev/null 2>&1 || true
+    wait "${APP_PID}" >/dev/null 2>&1 || true
+  fi
+  if [[ -n "${MOCK_PID}" ]] && kill -0 "${MOCK_PID}" >/dev/null 2>&1; then
+    kill "${MOCK_PID}" >/dev/null 2>&1 || true
+    wait "${MOCK_PID}" >/dev/null 2>&1 || true
+  fi
+  rm -rf "${TMP_DIR}"
+}
+trap cleanup EXIT
+
+cd "${ROOT_DIR}"
+
+echo "[selftest] gofmt"
+gofmt -w backend/main.go backend/handlers/api.go backend/models/db.go backend/services/cse.go backend/services/gemini.go backend/services/gemini_test.go
+
+echo "[selftest] python syntax"
+python3 -m py_compile worker/downloader.py scripts/mock_searxng.py
+
+echo "[selftest] go test"
+go test ./...
+
+echo "[selftest] go build"
+go build -o "${TMP_DIR}/ai-media-hub" ./backend
+
+echo "[selftest] start mock searxng"
+python3 scripts/mock_searxng.py --port "${MOCK_PORT}" >"${TMP_DIR}/mock-searxng.log" 2>&1 &
+MOCK_PID=$!
+
+echo "[selftest] start app"
+mkdir -p "${TMP_DIR}/downloads"
+APP_ROOT="${ROOT_DIR}" \
+APP_ADDR="127.0.0.1:${APP_PORT}" \
+SQLITE_PATH="${TMP_DIR}/media.db" \
+DOWNLOADS_DIR="${TMP_DIR}/downloads" \
+FRONTEND_DIR="${ROOT_DIR}/frontend" \
+WORKER_SCRIPT="${ROOT_DIR}/worker/downloader.py" \
+SEARXNG_BASE_URL="http://127.0.0.1:${MOCK_PORT}" \
+"${TMP_DIR}/ai-media-hub" >"${TMP_DIR}/app.log" 2>&1 &
+APP_PID=$!
+
+for _ in $(seq 1 30); do
+  if curl -fsS "http://127.0.0.1:${APP_PORT}/healthz" >/dev/null; then
+    break
+  fi
+  sleep 1
+done
+
+echo "[selftest] verify healthz"
+curl -fsS "http://127.0.0.1:${APP_PORT}/healthz" >"${TMP_DIR}/healthz.json"
+python3 - "${TMP_DIR}/healthz.json" <<'PY'
+import json
+import sys
+with open(sys.argv[1], "r", encoding="utf-8") as handle:
+    payload = json.load(handle)
+if payload.get("status") != "ok":
+    raise SystemExit(f"unexpected healthz payload: {payload}")
+PY
+
+echo "[selftest] verify search"
+curl -fsS \
+  -H "Content-Type: application/json" \
+  -d '{"query":"city rain","platforms":["envato","artgrid","google video"]}' \
+  "http://127.0.0.1:${APP_PORT}/api/search" >"${TMP_DIR}/search.json"
+python3 - "${TMP_DIR}/search.json" <<'PY'
+import json
+import sys
+with open(sys.argv[1], "r", encoding="utf-8") as handle:
+    payload = json.load(handle)
+results = payload.get("results") or []
+if len(results) < 2:
+    raise SystemExit(f"expected >= 2 search results, got {len(results)}: {payload}")
+if not all(item.get("link") for item in results):
+    raise SystemExit(f"search results must include links: {payload}")
+PY
+
+echo "[selftest] verify upload"
+printf 'selftest upload\n' > "${TMP_DIR}/sample.txt"
+curl -fsS -F "file=@${TMP_DIR}/sample.txt" "http://127.0.0.1:${APP_PORT}/api/upload" >"${TMP_DIR}/upload.json"
+python3 - "${TMP_DIR}/upload.json" "${TMP_DIR}/downloads" <<'PY'
+import json
+import os
+import sys
+with open(sys.argv[1], "r", encoding="utf-8") as handle:
+    payload = json.load(handle)
+filename = payload.get("filename")
+if not filename:
+    raise SystemExit(f"missing filename in upload payload: {payload}")
+target = os.path.join(sys.argv[2], filename)
+if not os.path.exists(target):
+    raise SystemExit(f"expected uploaded file at {target}")
+PY
+
+echo "[selftest] ok"