v0.2.7: 修复Redis连接 + 启动管理后台
- 修复Redis认证 (配置密码) - 启动Python管理后台 (端口9531, 15个功能开关) - 统一版本号 0.2.7 - 更新docker-compose.yml (镜像版本/Redis URL/Admin服务)
This commit is contained in:
344
source_clean/src/search/search.service.ts
Executable file
344
source_clean/src/search/search.service.ts
Executable file
@@ -0,0 +1,344 @@
|
||||
// Native fetch available in Node 20+
|
||||
import config from '../config';
|
||||
import { getDb } from '../database/database';
|
||||
import { localTimestamp } from '../utils/time';
|
||||
|
||||
export interface SearchResult {
|
||||
title: string;
|
||||
url: string;
|
||||
content: string;
|
||||
score?: number;
|
||||
source?: string;
|
||||
password?: string;
|
||||
datetime?: string;
|
||||
responseTimeMs?: number;
|
||||
}
|
||||
|
||||
export interface SearchResponse {
|
||||
results: SearchResult[];
|
||||
total: number;
|
||||
page: number;
|
||||
pageSize: number;
|
||||
}
|
||||
|
||||
export interface ApiSearchSource {
|
||||
name: string;
|
||||
url: string;
|
||||
method?: string; // GET | POST (default POST)
|
||||
headers?: Record<string, string>;
|
||||
body?: string; // JSON body template, supports {keyword} {page}
|
||||
resultPath: string; // dot-notation path to results array (e.g. "data.list")
|
||||
fieldMap: { // maps SearchResult fields to JSON response fields
|
||||
title?: string;
|
||||
url?: string;
|
||||
content?: string;
|
||||
password?: string;
|
||||
datetime?: string;
|
||||
};
|
||||
timeout?: number; // per-source timeout (ms), default 10000
|
||||
}
|
||||
|
||||
/** Simple dot/bracket notation JSON path accessor. */
|
||||
function jsonPathGet(obj: any, path: string): any {
|
||||
if (!obj || !path) return undefined;
|
||||
const parts = path
|
||||
.replace(/\[(\d+)\]/g, '.$1') // items[0] → items.0
|
||||
.split('.')
|
||||
.filter(Boolean);
|
||||
let current = obj;
|
||||
for (const part of parts) {
|
||||
if (current == null) return undefined;
|
||||
current = current[part];
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
/** Parse configured API search sources from system config. */
|
||||
function getApiSearchSources(): ApiSearchSource[] {
|
||||
try {
|
||||
const db = getDb();
|
||||
const raw = (db.prepare("SELECT value FROM system_configs WHERE key = 'api_search_sources'").get() as any)?.value;
|
||||
if (!raw) return [];
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed.filter((s: any) => s.url && s.resultPath);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Query a single API search source and return results with timing. */
|
||||
async function queryApiSource(
|
||||
source: ApiSearchSource,
|
||||
keyword: string,
|
||||
page: number,
|
||||
): Promise<{ source: string; results: SearchResult[]; responseTimeMs: number; error?: string }> {
|
||||
const startTime = Date.now();
|
||||
const timeout = source.timeout || 10000;
|
||||
const method = (source.method || 'POST').toUpperCase();
|
||||
|
||||
try {
|
||||
let url = source.url;
|
||||
const headers: Record<string, string> = { ...source.headers };
|
||||
|
||||
let body: string | undefined;
|
||||
if (source.body) {
|
||||
body = source.body
|
||||
.replace(/\{keyword\}/g, encodeURIComponent(keyword))
|
||||
.replace(/\{page\}/g, String(page));
|
||||
}
|
||||
|
||||
// For GET requests, append query params; for POST, use body
|
||||
const fetchOptions: RequestInit = {
|
||||
method,
|
||||
headers: { 'Content-Type': 'application/json', ...headers },
|
||||
signal: AbortSignal.timeout(timeout),
|
||||
};
|
||||
|
||||
if (method === 'GET') {
|
||||
// Parse body as JSON and convert to query string
|
||||
if (body) {
|
||||
try {
|
||||
const params = JSON.parse(body);
|
||||
const qs = new URLSearchParams(params).toString();
|
||||
url += (url.includes('?') ? '&' : '?') + qs;
|
||||
} catch {
|
||||
// If body is not JSON, append as raw query
|
||||
url += (url.includes('?') ? '&' : '?') + body;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(fetchOptions as any).body = body || JSON.stringify({ keyword, page });
|
||||
}
|
||||
|
||||
const response = await fetch(url, fetchOptions);
|
||||
const responseTimeMs = Date.now() - startTime;
|
||||
|
||||
if (!response.ok) {
|
||||
return { source: source.name, results: [], responseTimeMs, error: `HTTP ${response.status}` };
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const resultTimeMs = Date.now() - startTime;
|
||||
|
||||
// Extract results array using JSONPath
|
||||
const items = jsonPathGet(data, source.resultPath);
|
||||
if (!Array.isArray(items)) {
|
||||
return { source: source.name, results: [], responseTimeMs: resultTimeMs, error: 'resultPath not found or not an array' };
|
||||
}
|
||||
|
||||
// Map fields
|
||||
const fm = source.fieldMap || {};
|
||||
const results: SearchResult[] = items.map((item: any) => ({
|
||||
title: (fm.title ? item[fm.title] : item.title) || item.name || '',
|
||||
url: (fm.url ? item[fm.url] : item.url) || item.link || '',
|
||||
content: (fm.content ? item[fm.content] : item.content) || item.snippet || '',
|
||||
password: (fm.password ? item[fm.password] : item.password) || '',
|
||||
datetime: (fm.datetime ? item[fm.datetime] : item.datetime) || item.date || '',
|
||||
source: source.name,
|
||||
responseTimeMs: resultTimeMs,
|
||||
}));
|
||||
|
||||
return { source: source.name, results, responseTimeMs: resultTimeMs };
|
||||
} catch (err: any) {
|
||||
const responseTimeMs = Date.now() - startTime;
|
||||
return { source: source.name, results: [], responseTimeMs, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
/** Query all configured API search sources in parallel. */
|
||||
async function searchApiSources(keyword: string, page: number): Promise<{
|
||||
results: SearchResult[];
|
||||
sourceStats: { name: string; count: number; responseTimeMs: number; error?: string }[];
|
||||
}> {
|
||||
const sources = getApiSearchSources();
|
||||
if (sources.length === 0) return { results: [], sourceStats: [] };
|
||||
|
||||
const promises = sources.map(s => queryApiSource(s, keyword, page));
|
||||
const allResults = await Promise.all(promises);
|
||||
|
||||
const sourceStats = allResults.map(r => ({
|
||||
name: r.source,
|
||||
count: r.results.length,
|
||||
responseTimeMs: r.responseTimeMs,
|
||||
error: r.error,
|
||||
}));
|
||||
|
||||
// Merge all results, tag with source name, sort by response time (fastest first)
|
||||
const results = allResults
|
||||
.flatMap(r => r.results)
|
||||
.sort((a, b) => (a.responseTimeMs || 99999) - (b.responseTimeMs || 99999));
|
||||
|
||||
return { results, sourceStats };
|
||||
}
|
||||
|
||||
export async function search(keyword: string, page: number = 1, ip?: string): Promise<SearchResponse> {
|
||||
const db = getDb();
|
||||
const pansouUrl = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('pansou_url') as any)?.value || config.pansouUrl;
|
||||
const proxyEnabled = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('search_proxy_enabled') as any)?.value === 'true';
|
||||
const proxyUrl = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('search_proxy_url') as any)?.value || '';
|
||||
|
||||
// ── Run PanSou and API sources in parallel ──
|
||||
const pansouPromise = (async () => {
|
||||
const url = `${pansouUrl}/api/search`;
|
||||
const fetchOptions: any = {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ kw: keyword, page }),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
};
|
||||
const pansouStart = Date.now();
|
||||
const response = await fetch(url, fetchOptions);
|
||||
if (!response.ok) throw new Error(`PanSou API error: ${response.status}`);
|
||||
const data = await response.json() as any;
|
||||
return { data, responseTimeMs: Date.now() - pansouStart };
|
||||
})();
|
||||
|
||||
const apiSourcesPromise = searchApiSources(keyword, page);
|
||||
|
||||
const [pansouResult, apiSourcesResult] = await Promise.all([pansouPromise, apiSourcesPromise]);
|
||||
|
||||
const { data, responseTimeMs: pansouTime } = pansouResult;
|
||||
|
||||
// ── Parse PanSou results ──
|
||||
let items: any[] = [];
|
||||
let total = 0;
|
||||
|
||||
if (data.data?.merged_by_type) {
|
||||
for (const [cloudType, cloudItems] of Object.entries(data.data.merged_by_type)) {
|
||||
if (Array.isArray(cloudItems)) {
|
||||
items.push(...cloudItems.map((item: any) => ({
|
||||
...item,
|
||||
_cloud_type: cloudType,
|
||||
})));
|
||||
}
|
||||
}
|
||||
total = data.data.total || items.length;
|
||||
} else if (Array.isArray(data.data)) {
|
||||
items = data.data;
|
||||
total = data.total || items.length;
|
||||
} else if (Array.isArray(data.results)) {
|
||||
items = data.results;
|
||||
total = data.total || items.length;
|
||||
}
|
||||
|
||||
const pansouResults: SearchResult[] = items.map((item: any) => ({
|
||||
title: item.note || item.title || '',
|
||||
url: item.url || item.link || '',
|
||||
content: item.content || item.snippet || item.note || '',
|
||||
score: item.score || 0,
|
||||
source: item.source || item._cloud_type || 'pansou',
|
||||
password: item.password || '',
|
||||
datetime: item.datetime || '',
|
||||
responseTimeMs: pansouTime,
|
||||
images: item.images || [],
|
||||
}));
|
||||
|
||||
// ── Merge PanSou + API sources, sort by response time (fastest first) ──
|
||||
const allResults = [...apiSourcesResult.results, ...pansouResults]
|
||||
.sort((a, b) => (a.responseTimeMs || 99999) - (b.responseTimeMs || 99999));
|
||||
|
||||
// Deduplicate by URL within merged results
|
||||
const seenUrls = new Set<string>();
|
||||
const results: SearchResult[] = [];
|
||||
for (const r of allResults) {
|
||||
if (r.url && !seenUrls.has(r.url)) {
|
||||
seenUrls.add(r.url);
|
||||
results.push(r);
|
||||
} else if (!r.url) {
|
||||
results.push(r); // keep results without URLs (unlikely but safe)
|
||||
}
|
||||
}
|
||||
|
||||
total = results.length;
|
||||
|
||||
// Sort by datetime descending as secondary sort (preserve response-time groups)
|
||||
results.sort((a: any, b: any) => {
|
||||
const ta = a.datetime || '';
|
||||
const tb = b.datetime || '';
|
||||
if (!ta && !tb) return 0;
|
||||
if (!ta) return 1;
|
||||
if (!tb) return -1;
|
||||
return tb.localeCompare(ta);
|
||||
});
|
||||
|
||||
// Record search statistics
|
||||
recordSearchStats(keyword, results.length, ip);
|
||||
|
||||
// Update hot keywords
|
||||
updateHotKeywords(keyword);
|
||||
|
||||
return {
|
||||
results,
|
||||
total,
|
||||
page,
|
||||
pageSize: data.pageSize || 10,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply title filter rules to clean up search result titles.
|
||||
* Rules format (one per line):
|
||||
* # comment lines are ignored (hash must be followed by space)
|
||||
* /pattern/flags → regex: matched content is deleted from title
|
||||
* plain text → literal text: exact text is deleted from title wherever it appears
|
||||
*/
|
||||
export function applyTitleFilter(title: string, rules: string): string {
|
||||
if (!title || !rules) return title;
|
||||
const lines = rules.split('\n');
|
||||
let result = title;
|
||||
for (const rawLine of lines) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith('# ')) continue;
|
||||
try {
|
||||
if (line.startsWith('/') && line.lastIndexOf('/') > 0) {
|
||||
const lastSlashIdx = line.lastIndexOf('/');
|
||||
const pattern = line.substring(1, lastSlashIdx);
|
||||
const flags = line.substring(lastSlashIdx + 1);
|
||||
const anchored = pattern.startsWith('^') ? pattern : '^' + pattern;
|
||||
const re = new RegExp(anchored, flags);
|
||||
const match = re.exec(result);
|
||||
if (match && match.index === 0) {
|
||||
result = result.slice(match[0].length);
|
||||
}
|
||||
} else {
|
||||
if (result.startsWith(line)) {
|
||||
result = result.slice(line.length);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return result.trim();
|
||||
}
|
||||
|
||||
function recordSearchStats(keyword: string, resultCount: number, ip?: string): void {
|
||||
try {
|
||||
const db = getDb();
|
||||
db.prepare(
|
||||
'INSERT INTO search_stats (keyword, intent, result_count, ip_address, created_at) VALUES (?, ?, ?, ?, ?)'
|
||||
).run(keyword, 'SEARCH', resultCount, ip || '', localTimestamp());
|
||||
} catch (err) {
|
||||
console.error('[Search] Failed to record stats:', err);
|
||||
}
|
||||
}
|
||||
|
||||
function updateHotKeywords(keyword: string): void {
|
||||
try {
|
||||
const db = getDb();
|
||||
const existing = db.prepare('SELECT id FROM hot_keywords WHERE keyword = ?').get(keyword) as any;
|
||||
|
||||
if (existing) {
|
||||
db.prepare(
|
||||
"UPDATE hot_keywords SET search_count = search_count + 1, updated_at = ? WHERE keyword = ?"
|
||||
).run(localTimestamp(), keyword);
|
||||
} else {
|
||||
db.prepare(
|
||||
"INSERT INTO hot_keywords (keyword, search_count, updated_at) VALUES (?, 1, ?)"
|
||||
).run(keyword, localTimestamp());
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Search] Failed to update hot keywords:', err);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user