diff --git a/TODO.md b/TODO.md index 085ab7a..5a0f2ba 100644 --- a/TODO.md +++ b/TODO.md @@ -341,6 +341,17 @@ - [ ] full browser-level validation was not fully reproducible in this environment ## Recent Change Log +- Date: `2026-03-16` +- What changed: + - Added `/api/preview/stream` so remote preview assets are fetched through the backend instead of relying on the browser to load Envato / Artgrid media directly. + - MP4 previews are cached on disk under the downloads area, and HLS playlists are rewritten so segment fetches also flow through the same backend proxy route. +- Why it changed: + - Direct browser loading of remote preview URLs was still unstable and often failed due to upstream restrictions or missing headers. +- How it was verified: + - code-path inspection of preview proxy and playlist rewrite flow +- What is still risky or incomplete: + - HLS caching is not yet persisted segment-by-segment; current implementation rewrites playlists and proxies segment requests live. + - Date: `2026-03-16` - What changed: - Relaxed final recommendation merge so Gemini-reviewed non-negative items can still appear, and only a small preview-capable ranked filler set is used when the result list is otherwise too thin. diff --git a/backend/handlers/api.go b/backend/handlers/api.go index ae97522..4d6159d 100644 --- a/backend/handlers/api.go +++ b/backend/handlers/api.go @@ -2,11 +2,14 @@ package handlers import ( "bufio" + "crypto/sha1" "database/sql" "encoding/json" "errors" "fmt" + "io" "net/http" + "net/url" "os" "os/exec" "path/filepath" @@ -24,12 +27,13 @@ import ( ) type App struct { - DB *sql.DB - DownloadsDir string - WorkerScript string - SearchService *services.SearchService - GeminiService *services.GeminiService - Hub *Hub + DB *sql.DB + DownloadsDir string + PreviewCacheDir string + WorkerScript string + SearchService *services.SearchService + GeminiService *services.GeminiService + Hub *Hub } type Hub struct { @@ -92,6 +96,7 @@ func RegisterRoutes(router *gin.Engine, app *App) { }) router.GET("/ws", app.handleWS) router.GET("/api/history/check", app.checkDuplicate) + router.GET("/api/preview/stream", app.streamPreview) router.POST("/api/download/preview", app.previewDownload) router.POST("/api/upload", app.uploadFile) router.POST("/api/download", app.startDownload) @@ -134,6 +139,61 @@ func (a *App) checkDuplicate(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"exists": record != nil, "record": record}) } +func (a *App) streamPreview(c *gin.Context) { + target := strings.TrimSpace(c.Query("url")) + if target == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "url is required"}) + return + } + + req, err := http.NewRequest(http.MethodGet, target, nil) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Referer", inferPreviewReferer(target)) + + resp, err := a.SearchService.Client.Do(req) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + defer resp.Body.Close() + + if resp.StatusCode >= 300 { + c.JSON(http.StatusBadGateway, gin.H{"error": fmt.Sprintf("preview source returned %d", resp.StatusCode)}) + return + } + + contentType := resp.Header.Get("Content-Type") + if strings.Contains(strings.ToLower(target), ".m3u8") || strings.Contains(strings.ToLower(contentType), "mpegurl") { + body, err := io.ReadAll(io.LimitReader(resp.Body, 2*1024*1024)) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + rewritten := rewriteM3U8Playlist(string(body), target) + c.Header("Content-Type", "application/vnd.apple.mpegurl") + c.String(http.StatusOK, rewritten) + return + } + + if isCacheablePreview(target, contentType) { + if cachedPath, err := a.cachePreviewResponse(target, contentType, resp.Body); err == nil { + c.File(cachedPath) + return + } + } + + if contentType != "" { + c.Header("Content-Type", contentType) + } + c.Status(http.StatusOK) + _, _ = io.Copy(c.Writer, resp.Body) +} + func (a *App) uploadFile(c *gin.Context) { file, err := c.FormFile("file") if err != nil { @@ -562,6 +622,68 @@ func EnsurePaths(downloadsDir, workerScript string) error { return nil } +func inferPreviewReferer(target string) string { + lower := strings.ToLower(target) + switch { + case strings.Contains(lower, "envatousercontent.com"), strings.Contains(lower, "elements.envato.com"): + return "https://elements.envato.com/" + case strings.Contains(lower, "artgrid"), strings.Contains(lower, "artlist"): + return "https://artgrid.io/" + default: + return "" + } +} + +func rewriteM3U8Playlist(body, target string) string { + lines := strings.Split(body, "\n") + baseURL := target + if idx := strings.LastIndex(baseURL, "/"); idx >= 0 { + baseURL = baseURL[:idx+1] + } + for idx, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + resolved := trimmed + if !strings.HasPrefix(trimmed, "http://") && !strings.HasPrefix(trimmed, "https://") { + resolved = baseURL + strings.TrimPrefix(trimmed, "/") + } + lines[idx] = "/api/preview/stream?url=" + url.QueryEscape(resolved) + } + return strings.Join(lines, "\n") +} + +func isCacheablePreview(target, contentType string) bool { + lower := strings.ToLower(target + " " + contentType) + return strings.Contains(lower, ".mp4") || strings.Contains(lower, "video/mp4") +} + +func (a *App) cachePreviewResponse(target, contentType string, body io.Reader) (string, error) { + if a.PreviewCacheDir == "" { + return "", fmt.Errorf("preview cache dir is not configured") + } + if err := os.MkdirAll(a.PreviewCacheDir, 0o755); err != nil { + return "", err + } + + sum := sha1.Sum([]byte(target)) + ext := ".bin" + if strings.Contains(strings.ToLower(target), ".mp4") || strings.Contains(strings.ToLower(contentType), "video/mp4") { + ext = ".mp4" + } + path := filepath.Join(a.PreviewCacheDir, fmt.Sprintf("%x%s", sum, ext)) + file, err := os.Create(path) + if err != nil { + return "", err + } + defer file.Close() + if _, err := io.Copy(file, body); err != nil { + return "", err + } + return path, nil +} + func summarizeOutput(prefix string, output []byte, err error) string { trimmed := strings.TrimSpace(string(output)) if trimmed == "" && err != nil { diff --git a/backend/main.go b/backend/main.go index 9b9269a..6d5507f 100644 --- a/backend/main.go +++ b/backend/main.go @@ -31,9 +31,10 @@ func main() { } app := &handlers.App{ - DB: db, - DownloadsDir: downloadsDir, - WorkerScript: workerScript, + DB: db, + DownloadsDir: downloadsDir, + PreviewCacheDir: filepath.Join(downloadsDir, ".preview-cache"), + WorkerScript: workerScript, SearchService: services.NewSearchService( os.Getenv("SEARXNG_BASE_URL"), os.Getenv("SEARXNG_GOOGLE_VIDEO_ENGINE"), diff --git a/frontend/app.js b/frontend/app.js index d8f7c53..233b959 100644 --- a/frontend/app.js +++ b/frontend/app.js @@ -75,6 +75,13 @@ const hlsInstances = new WeakMap(); const debugEntries = []; const PREVIEW_PLACEHOLDER = "https://placehold.co/1280x720/0a0a0a/ffffff?text=Preview"; +function proxiedPreviewURL(src) { + if (!src) { + return ""; + } + return `/api/preview/stream?url=${encodeURIComponent(src)}`; +} + function setStatus(label, progress) { statusLabel.textContent = label; statusBar.style.width = `${Math.max(0, Math.min(100, progress))}%`; @@ -418,7 +425,7 @@ function showResultModalVideo(src) { if (!src) { return; } - attachVideoSource(resultModalVideo, src); + attachVideoSource(resultModalVideo, proxiedPreviewURL(src)); setHidden(resultModalVideo, false, ""); } @@ -452,7 +459,7 @@ function renderResults(results) { mediaArea.addEventListener("mouseenter", () => { logEvent("preview:hover:start", { title: item.title, source: item.source, previewVideoUrl: item.previewVideoUrl }); overlays.forEach((overlay) => overlay.classList.add("hidden")); - startHoverPreview(previewVideo, item.previewVideoUrl); + startHoverPreview(previewVideo, proxiedPreviewURL(item.previewVideoUrl)); }); mediaArea.addEventListener("mouseleave", () => { logEvent("preview:hover:end", { title: item.title, source: item.source }); @@ -568,7 +575,7 @@ function openPreviewModal(preview) { previewThumbnail.alt = preview.title; resetPreviewPlayer(); if (preview.previewStreamUrl) { - attachVideoSource(previewVideo, preview.previewStreamUrl); + attachVideoSource(previewVideo, proxiedPreviewURL(preview.previewStreamUrl)); setHidden(previewVideo, false, ""); setHidden(previewThumbnail, true, ""); } else {