CloudSearch/source_clean/src/search/search.service.ts

// Native fetch available in Node 20+
import config from '../config';
import { getDb } from '../database/database';
import { localTimestamp } from '../utils/time';

export interface SearchResult {
  title: string;
  url: string;
  content: string;
  score?: number;
  source?: string;
  password?: string;
  datetime?: string;
  responseTimeMs?: number;
}

export interface SearchResponse {
  results: SearchResult[];
  total: number;
  page: number;
  pageSize: number;
}

export interface ApiSearchSource {
  name: string;
  url: string;
  method?: string;              // GET | POST (default POST)
  headers?: Record<string, string>;
  body?: string;                // JSON body template, supports {keyword} {page}
  resultPath: string;           // dot-notation path to results array (e.g. "data.list")
  fieldMap: {                   // maps SearchResult fields to JSON response fields
    title?: string;
    url?: string;
    content?: string;
    password?: string;
    datetime?: string;
  };
  timeout?: number;             // per-source timeout (ms), default 10000
}

/** Simple dot/bracket notation JSON path accessor. */
function jsonPathGet(obj: any, path: string): any {
  if (!obj || !path) return undefined;
  const parts = path
    .replace(/\[(\d+)\]/g, '.$1')   // items[0] → items.0
    .split('.')
    .filter(Boolean);
  let current = obj;
  for (const part of parts) {
    if (current == null) return undefined;
    current = current[part];
  }
  return current;
}

/** Parse configured API search sources from system config. */
function getApiSearchSources(): ApiSearchSource[] {
  try {
    const db = getDb();
    const raw = (db.prepare("SELECT value FROM system_configs WHERE key = 'api_search_sources'").get() as any)?.value;
    if (!raw) return [];
    const parsed = JSON.parse(raw);
    if (!Array.isArray(parsed)) return [];
    return parsed.filter((s: any) => s.url && s.resultPath);
  } catch {
    return [];
  }
}

/** Query a single API search source and return results with timing. */
async function queryApiSource(
  source: ApiSearchSource,
  keyword: string,
  page: number,
): Promise<{ source: string; results: SearchResult[]; responseTimeMs: number; error?: string }> {
  const startTime = Date.now();
  const timeout = source.timeout || 10000;
  const method = (source.method || 'POST').toUpperCase();

  try {
    let url = source.url;
    const headers: Record<string, string> = { ...source.headers };

    let body: string | undefined;
    if (source.body) {
      body = source.body
        .replace(/\{keyword\}/g, encodeURIComponent(keyword))
        .replace(/\{page\}/g, String(page));
    }

    // For GET requests, append query params; for POST, use body
    const fetchOptions: RequestInit = {
      method,
      headers: { 'Content-Type': 'application/json', ...headers },
      signal: AbortSignal.timeout(timeout),
    };

    if (method === 'GET') {
      // Parse body as JSON and convert to query string
      if (body) {
        try {
          const params = JSON.parse(body);
          const qs = new URLSearchParams(params).toString();
          url += (url.includes('?') ? '&' : '?') + qs;
        } catch {
          // If body is not JSON, append as raw query
          url += (url.includes('?') ? '&' : '?') + body;
        }
      }
    } else {
      (fetchOptions as any).body = body || JSON.stringify({ keyword, page });
    }

    const response = await fetch(url, fetchOptions);
    const responseTimeMs = Date.now() - startTime;

    if (!response.ok) {
      return { source: source.name, results: [], responseTimeMs, error: `HTTP ${response.status}` };
    }

    const data = await response.json();
    const resultTimeMs = Date.now() - startTime;

    // Extract results array using JSONPath
    const items = jsonPathGet(data, source.resultPath);
    if (!Array.isArray(items)) {
      return { source: source.name, results: [], responseTimeMs: resultTimeMs, error: 'resultPath not found or not an array' };
    }

    // Map fields
    const fm = source.fieldMap || {};
    const results: SearchResult[] = items.map((item: any) => ({
      title: (fm.title ? item[fm.title] : item.title) || item.name || '',
      url: (fm.url ? item[fm.url] : item.url) || item.link || '',
      content: (fm.content ? item[fm.content] : item.content) || item.snippet || '',
      password: (fm.password ? item[fm.password] : item.password) || '',
      datetime: (fm.datetime ? item[fm.datetime] : item.datetime) || item.date || '',
      source: source.name,
      responseTimeMs: resultTimeMs,
    }));

    return { source: source.name, results, responseTimeMs: resultTimeMs };
  } catch (err: any) {
    const responseTimeMs = Date.now() - startTime;
    return { source: source.name, results: [], responseTimeMs, error: err.message };
  }
}

/** Query all configured API search sources in parallel. */
async function searchApiSources(keyword: string, page: number): Promise<{
  results: SearchResult[];
  sourceStats: { name: string; count: number; responseTimeMs: number; error?: string }[];
}> {
  const sources = getApiSearchSources();
  if (sources.length === 0) return { results: [], sourceStats: [] };

  const promises = sources.map(s => queryApiSource(s, keyword, page));
  const allResults = await Promise.all(promises);

  const sourceStats = allResults.map(r => ({
    name: r.source,
    count: r.results.length,
    responseTimeMs: r.responseTimeMs,
    error: r.error,
  }));

  // Merge all results, tag with source name, sort by response time (fastest first)
  const results = allResults
    .flatMap(r => r.results)
    .sort((a, b) => (a.responseTimeMs || 99999) - (b.responseTimeMs || 99999));

  return { results, sourceStats };
}

export async function search(keyword: string, page: number = 1, ip?: string): Promise<SearchResponse> {
  const db = getDb();
  const pansouUrl = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('pansou_url') as any)?.value || config.pansouUrl;
  const proxyEnabled = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('search_proxy_enabled') as any)?.value === 'true';
  const proxyUrl = (db.prepare('SELECT value FROM system_configs WHERE key = ?').get('search_proxy_url') as any)?.value || '';

  // ── Run PanSou and API sources in parallel ──
  const pansouPromise = (async () => {
    const url = `${pansouUrl}/api/search`;
    const fetchOptions: any = {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ kw: keyword, page }),
      signal: AbortSignal.timeout(10000),
    };
    const pansouStart = Date.now();
    const response = await fetch(url, fetchOptions);
    if (!response.ok) throw new Error(`PanSou API error: ${response.status}`);
    const data = await response.json() as any;
    return { data, responseTimeMs: Date.now() - pansouStart };
  })();

  const apiSourcesPromise = searchApiSources(keyword, page);

  const [pansouResult, apiSourcesResult] = await Promise.all([pansouPromise, apiSourcesPromise]);

  const { data, responseTimeMs: pansouTime } = pansouResult;

  // ── Parse PanSou results ──
  let items: any[] = [];
  let total = 0;

  if (data.data?.merged_by_type) {
    for (const [cloudType, cloudItems] of Object.entries(data.data.merged_by_type)) {
      if (Array.isArray(cloudItems)) {
        items.push(...cloudItems.map((item: any) => ({
          ...item,
          _cloud_type: cloudType,
        })));
      }
    }
    total = data.data.total || items.length;
  } else if (Array.isArray(data.data)) {
    items = data.data;
    total = data.total || items.length;
  } else if (Array.isArray(data.results)) {
    items = data.results;
    total = data.total || items.length;
  }

  const pansouResults: SearchResult[] = items.map((item: any) => ({
    title: item.note || item.title || '',
    url: item.url || item.link || '',
    content: item.content || item.snippet || item.note || '',
    score: item.score || 0,
    source: item.source || item._cloud_type || 'pansou',
    password: item.password || '',
    datetime: item.datetime || '',
    responseTimeMs: pansouTime,
    images: item.images || [],
  }));

  // ── Merge PanSou + API sources, sort by response time (fastest first) ──
  const allResults = [...apiSourcesResult.results, ...pansouResults]
    .sort((a, b) => (a.responseTimeMs || 99999) - (b.responseTimeMs || 99999));

  // Deduplicate by URL within merged results
  const seenUrls = new Set<string>();
  const results: SearchResult[] = [];
  for (const r of allResults) {
    if (r.url && !seenUrls.has(r.url)) {
      seenUrls.add(r.url);
      results.push(r);
    } else if (!r.url) {
      results.push(r);  // keep results without URLs (unlikely but safe)
    }
  }

  total = results.length;

  // Sort by datetime descending as secondary sort (preserve response-time groups)
  results.sort((a: any, b: any) => {
    const ta = a.datetime || '';
    const tb = b.datetime || '';
    if (!ta && !tb) return 0;
    if (!ta) return 1;
    if (!tb) return -1;
    return tb.localeCompare(ta);
  });

  // Record search statistics
  recordSearchStats(keyword, results.length, ip);

  // Update hot keywords
  updateHotKeywords(keyword);

  return {
    results,
    total,
    page,
    pageSize: data.pageSize || 10,
  };
}

/**
 * Apply title filter rules to clean up search result titles.
 * Rules format (one per line):
 *   #  comment lines are ignored (hash must be followed by space)
 *   /pattern/flags  → regex: matched content is deleted from title
 *   plain text      → literal text: exact text is deleted from title wherever it appears
 */
export function applyTitleFilter(title: string, rules: string): string {
  if (!title || !rules) return title;
  const lines = rules.split('\n');
  let result = title;
  for (const rawLine of lines) {
    const line = rawLine.trim();
    if (!line || line.startsWith('# ')) continue;
    try {
      if (line.startsWith('/') && line.lastIndexOf('/') > 0) {
        const lastSlashIdx = line.lastIndexOf('/');
        const pattern = line.substring(1, lastSlashIdx);
        const flags = line.substring(lastSlashIdx + 1);
        const anchored = pattern.startsWith('^') ? pattern : '^' + pattern;
        const re = new RegExp(anchored, flags);
        const match = re.exec(result);
        if (match && match.index === 0) {
          result = result.slice(match[0].length);
        }
      } else {
        if (result.startsWith(line)) {
          result = result.slice(line.length);
        }
      }
    } catch {
      continue;
    }
  }
  return result.trim();
}

function recordSearchStats(keyword: string, resultCount: number, ip?: string): void {
  try {
    const db = getDb();
    db.prepare(
      'INSERT INTO search_stats (keyword, intent, result_count, ip_address, created_at) VALUES (?, ?, ?, ?, ?)'
    ).run(keyword, 'SEARCH', resultCount, ip || '', localTimestamp());
  } catch (err) {
    console.error('[Search] Failed to record stats:', err);
  }
}

function updateHotKeywords(keyword: string): void {
  try {
    const db = getDb();
    const existing = db.prepare('SELECT id FROM hot_keywords WHERE keyword = ?').get(keyword) as any;

    if (existing) {
      db.prepare(
        "UPDATE hot_keywords SET search_count = search_count + 1, updated_at = ? WHERE keyword = ?"
      ).run(localTimestamp(), keyword);
    } else {
      db.prepare(
        "INSERT INTO hot_keywords (keyword, search_count, updated_at) VALUES (?, 1, ?)"
      ).run(keyword, localTimestamp());
    }
  } catch (err) {
    console.error('[Search] Failed to update hot keywords:', err);
  }
}