// Native fetch available in Node 20+ import config from '../config'; import { getDb } from '../database/database'; import { localTimestamp } from '../utils/time'; export interface SearchResult { title: string; url: string; content: string; score?: number; source?: string; password?: string; datetime?: string; responseTimeMs?: number; } export interface SearchResponse { results: SearchResult[]; total: number; page: number; pageSize: number; } export interface ApiSearchSource { name: string; url: string; method?: string; // GET | POST (default POST) headers?: Record; body?: string; // JSON body template, supports {keyword} {page} resultPath: string; // dot-notation path to results array (e.g. "data.list") fieldMap: { // maps SearchResult fields to JSON response fields title?: string; url?: string; content?: string; password?: string; datetime?: string; }; timeout?: number; // per-source timeout (ms), default 10000 } /** Simple dot/bracket notation JSON path accessor. */ function jsonPathGet(obj: any, path: string): any { if (!obj || !path) return undefined; const parts = path .replace(/\[(\d+)\]/g, '.$1') // items[0] → items.0 .split('.') .filter(Boolean); let current = obj; for (const part of parts) { if (current == null) return undefined; current = current[part]; } return current; } /** Parse configured API search sources from system config. */ function getApiSearchSources(): ApiSearchSource[] { try { const db = getDb(); const raw = (db.prepare("SELECT value FROM system_configs WHERE key = 'api_search_sources'").get() as any)?.value; if (!raw) return []; const parsed = JSON.parse(raw); if (!Array.isArray(parsed)) return []; return parsed.filter((s: any) => s.url && s.resultPath); } catch { return []; } } /** Query a single API search source and return results with timing. */ async function queryApiSource( source: ApiSearchSource, keyword: string, page: number, ): Promise<{ source: string; results: SearchResult[]; responseTimeMs: number; error?: string }> { const startTime = Date.now(); const timeout = source.timeout || 10000; const method = (source.method || 'POST').toUpperCase(); try { let url = source.url; const headers: Record = { ...source.headers }; let body: string | undefined; if (source.body) { body = source.body .replace(/\{keyword\}/g, encodeURIComponent(keyword)) .replace(/\{page\}/g, String(page)); } // For GET requests, append query params; for POST, use body const fetchOptions: RequestInit = { method, headers: { 'Content-Type': 'application/json', ...headers }, signal: AbortSignal.timeout(timeout), }; if (method === 'GET') { // Parse body as JSON and convert to query string if (body) { try { const params = JSON.parse(body); const qs = new URLSearchParams(params).toString(); url += (url.includes('?') ? '&' : '?') + qs; } catch { // If body is not JSON, append as raw query url += (url.includes('?') ? '&' : '?') + body; } } } else { (fetchOptions as any).body = body || JSON.stringify({ keyword, page }); } const response = await fetch(url, fetchOptions); const responseTimeMs = Date.now() - startTime; if (!response.ok) { return { source: source.name, results: [], responseTimeMs, error: `HTTP ${response.status}` }; } const data = await response.json(); const resultTimeMs = Date.now() - startTime; // Extract results array using JSONPath const items = jsonPathGet(data, source.resultPath); if (!Array.isArray(items)) { return { source: source.name, results: [], responseTimeMs: resultTimeMs, error: 'resultPath not found or not an array' }; } // Map fields const fm = source.fieldMap || {}; const results: SearchResult[] = items.map((item: any) => ({ title: (fm.title ? item[fm.title] : item.title) || item.name || '', url: (fm.url ? item[fm.url] : item.url) || item.link || '', content: (fm.content ? item[fm.content] : item.content) || item.snippet || '', password: (fm.password ? item[fm.password] : item.password) || '', datetime: (fm.datetime ? item[fm.datetime] : item.datetime) || item.date || '', source: source.name, responseTimeMs: resultTimeMs, })); return { source: source.name, results, responseTimeMs: resultTimeMs }; } catch (err: any) { const responseTimeMs = Date.now() - startTime; return { source: source.name, results: [], responseTimeMs, error: err.message }; } } /** Query all configured API search sources in parallel. */ async function searchApiSources(keyword: string, page: number): Promise<{ results: SearchResult[]; sourceStats: { name: string; count: number; responseTimeMs: number; error?: string }[]; }> { const sources = getApiSearchSources(); if (sources.length === 0) return { results: [], sourceStats: [] }; const promises = sources.map(s => queryApiSource(s, keyword, page)); const allResults = await Promise.all(promises); const sourceStats = allResults.map(r => ({ name: r.source, count: r.results.length, responseTimeMs: r.responseTimeMs, error: r.error, })); // Merge all results, tag with source name, sort by response time (fastest first) const results = allResults .flatMap(r => r.results) .sort((a, b) => (a.responseTimeMs || 99999) - (b.responseTimeMs || 99999)); return { results, sourceStats }; } export async function search(keyword: string, page: number = 1, ip?: string): Promise { const db = getDb(); const pansouUrl = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('pansou_url') as any)?.value || config.pansouUrl; const proxyEnabled = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('search_proxy_enabled') as any)?.value === 'true'; const proxyUrl = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('search_proxy_url') as any)?.value || ''; // ── Run PanSou and API sources in parallel ── const pansouPromise = (async () => { const url = `${pansouUrl}/api/search`; const fetchOptions: any = { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ kw: keyword, page }), signal: AbortSignal.timeout(10000), }; const pansouStart = Date.now(); const response = await fetch(url, fetchOptions); if (!response.ok) throw new Error(`PanSou API error: ${response.status}`); const data = await response.json() as any; return { data, responseTimeMs: Date.now() - pansouStart }; })(); const apiSourcesPromise = searchApiSources(keyword, page); const [pansouResult, apiSourcesResult] = await Promise.all([pansouPromise, apiSourcesPromise]); const { data, responseTimeMs: pansouTime } = pansouResult; // ── Parse PanSou results ── let items: any[] = []; let total = 0; if (data.data?.merged_by_type) { for (const [cloudType, cloudItems] of Object.entries(data.data.merged_by_type)) { if (Array.isArray(cloudItems)) { items.push(...cloudItems.map((item: any) => ({ ...item, _cloud_type: cloudType, }))); } } total = data.data.total || items.length; } else if (Array.isArray(data.data)) { items = data.data; total = data.total || items.length; } else if (Array.isArray(data.results)) { items = data.results; total = data.total || items.length; } const pansouResults: SearchResult[] = items.map((item: any) => ({ title: item.note || item.title || '', url: item.url || item.link || '', content: item.content || item.snippet || item.note || '', score: item.score || 0, source: item.source || item._cloud_type || 'pansou', password: item.password || '', datetime: item.datetime || '', responseTimeMs: pansouTime, images: item.images || [], })); // ── Merge PanSou + API sources, sort by response time (fastest first) ── const allResults = [...apiSourcesResult.results, ...pansouResults] .sort((a, b) => (a.responseTimeMs || 99999) - (b.responseTimeMs || 99999)); // Deduplicate by URL within merged results const seenUrls = new Set(); const results: SearchResult[] = []; for (const r of allResults) { if (r.url && !seenUrls.has(r.url)) { seenUrls.add(r.url); results.push(r); } else if (!r.url) { results.push(r); // keep results without URLs (unlikely but safe) } } total = results.length; // Sort by datetime descending as secondary sort (preserve response-time groups) results.sort((a: any, b: any) => { const ta = a.datetime || ''; const tb = b.datetime || ''; if (!ta && !tb) return 0; if (!ta) return 1; if (!tb) return -1; return tb.localeCompare(ta); }); // Record search statistics recordSearchStats(keyword, results.length, ip); // Update hot keywords updateHotKeywords(keyword); return { results, total, page, pageSize: data.pageSize || 10, }; } /** * Apply title filter rules to clean up search result titles. * Rules format (one per line): * # comment lines are ignored (hash must be followed by space) * /pattern/flags → regex: matched content is deleted from title * plain text → literal text: exact text is deleted from title wherever it appears */ export function applyTitleFilter(title: string, rules: string): string { if (!title || !rules) return title; const lines = rules.split('\n'); let result = title; for (const rawLine of lines) { const line = rawLine.trim(); if (!line || line.startsWith('# ')) continue; try { if (line.startsWith('/') && line.lastIndexOf('/') > 0) { const lastSlashIdx = line.lastIndexOf('/'); const pattern = line.substring(1, lastSlashIdx); const flags = line.substring(lastSlashIdx + 1); const anchored = pattern.startsWith('^') ? pattern : '^' + pattern; const re = new RegExp(anchored, flags); const match = re.exec(result); if (match && match.index === 0) { result = result.slice(match[0].length); } } else { if (result.startsWith(line)) { result = result.slice(line.length); } } } catch { continue; } } return result.trim(); } function recordSearchStats(keyword: string, resultCount: number, ip?: string): void { try { const db = getDb(); db.prepare( 'INSERT INTO search_stats (keyword, intent, result_count, ip_address, created_at) VALUES (?, ?, ?, ?, ?)' ).run(keyword, 'SEARCH', resultCount, ip || '', localTimestamp()); } catch (err) { console.error('[Search] Failed to record stats:', err); } } function updateHotKeywords(keyword: string): void { try { const db = getDb(); const existing = db.prepare('SELECT id FROM hot_keywords WHERE keyword = ?').get(keyword) as any; if (existing) { db.prepare( "UPDATE hot_keywords SET search_count = search_count + 1, updated_at = ? WHERE keyword = ?" ).run(localTimestamp(), keyword); } else { db.prepare( "INSERT INTO hot_keywords (keyword, search_count, updated_at) VALUES (?, 1, ?)" ).run(keyword, localTimestamp()); } } catch (err) { console.error('[Search] Failed to update hot keywords:', err); } }