From bcaa2a320cfe59d9f47df219649a40d3017a1e31 Mon Sep 17 00:00:00 2001
From: Hermes Agent <hermes-agent@local>
Date: Sat, 23 May 2026 19:01:18 +0000
Subject: [PATCH] Add 8h continuous collector runner

---
 run_8h_collect.sh | 100 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100755 run_8h_collect.sh

diff --git a/run_8h_collect.sh b/run_8h_collect.sh
new file mode 100755
index 0000000..35a65e6
--- /dev/null
+++ b/run_8h_collect.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+set -u
+
+cd /opt/data/workspace/qinglong
+
+START_TS=$(date +%s)
+END_TS=$((START_TS + 8*60*60))
+ROUND=0
+LOG_DIR="logs"
+mkdir -p "$LOG_DIR"
+LOG_FILE="$LOG_DIR/collect_8h_$(date +%Y%m%d_%H%M%S).log"
+
+log() {
+  echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG_FILE"
+}
+
+commit_if_changed() {
+  if [[ -n "$(git status --short)" ]]; then
+    log "[GIT] changes detected, committing..."
+    git add . >>"$LOG_FILE" 2>&1
+    git commit -m "Collect Qinglong scripts batch $(date '+%Y-%m-%d %H:%M')" >>"$LOG_FILE" 2>&1 || true
+    git push origin main >>"$LOG_FILE" 2>&1 || log "[GIT] push failed; will retry next round"
+  else
+    log "[GIT] no changes"
+  fi
+}
+
+count_records() {
+  python3 - <<'PY' 2>/dev/null || echo 0
+import json
+try:
+    print(len(json.load(open('INDEX.json', encoding='utf-8'))))
+except Exception:
+    print(0)
+PY
+}
+
+log "[START] 8h collector started"
+log "[INFO] repo=$(git remote get-url origin | sed 's#oauth2:[^@]*@#oauth2:***@#')"
+log "[INFO] head=$(git rev-parse --short HEAD 2>/dev/null || true)"
+
+while [[ $(date +%s) -lt $END_TS ]]; do
+  ROUND=$((ROUND + 1))
+  NOW=$(date +%s)
+  LEFT=$((END_TS - NOW))
+  BEFORE=$(count_records)
+  log "========== ROUND $ROUND START | left=${LEFT}s | records_before=${BEFORE} =========="
+
+  # 低速采集：让 GitHub 限流后可等待更久；公开网页搜索开启；京东过滤由 collector.py 内置。
+  MAX_REPOS_PER_QUERY=4 \
+  MAX_FILES_TOTAL=180 \
+  REQUEST_SLEEP=1.2 \
+  ENABLE_WEB_SEARCH=true \
+  GITHUB_RETRY_MAX_WAIT=2400 \
+  uv run --with requests python -u collector.py >>"$LOG_FILE" 2>&1 || log "[WARN] collector exited non-zero; continue next round"
+
+  # 重建索引，确保中断/限流后的落盘文件也进入索引，同时再次执行京东过滤。
+  uv run --with requests python -u -c "import collector; collector.rebuild_index_from_files()" >>"$LOG_FILE" 2>&1 || log "[WARN] rebuild index failed"
+
+  AFTER=$(count_records)
+  log "[ROUND $ROUND] records_after=${AFTER} delta=$((AFTER - BEFORE))"
+
+  # 简单扫描京东关键词，若发现则记录；collector 重建时会尽量删除命中文件。
+  python3 - <<'PY' >>"$LOG_FILE" 2>&1 || true
+import os
+kws=['京东','jd_','JD_COOKIE','pt_key','pt_pin','api.m.jd.com','m.jd.com','jingdong','jd.com']
+hits=[]
+for root,_,files in os.walk('脚本库'):
+    for f in files:
+        p=os.path.join(root,f)
+        try:
+            txt=open(p,encoding='utf-8',errors='ignore').read(50000)
+        except Exception:
+            continue
+        hay=(p+'\n'+txt).lower()
+        if any(k.lower() in hay for k in kws):
+            hits.append(p)
+print('[JD_SCAN] hits=', len(hits))
+for h in hits[:20]: print('[JD_SCAN_HIT]', h)
+PY
+
+  commit_if_changed
+
+  NOW=$(date +%s)
+  if [[ $NOW -ge $END_TS ]]; then
+    break
+  fi
+
+  # 慢速循环：每轮后休眠 12-20 分钟，避免搜索端限流；剩余时间不足则按剩余时间睡。
+  SLEEP=$((720 + RANDOM % 480))
+  LEFT=$((END_TS - NOW))
+  if [[ $SLEEP -gt $LEFT ]]; then SLEEP=$LEFT; fi
+  log "[SLEEP] ${SLEEP}s"
+  sleep "$SLEEP"
+done
+
+log "[FINAL] rebuilding index and pushing final state"
+uv run --with requests python -u -c "import collector; collector.rebuild_index_from_files()" >>"$LOG_FILE" 2>&1 || true
+commit_if_changed
+log "[DONE] 8h collector finished | records=$(count_records) | head=$(git rev-parse --short HEAD 2>/dev/null || true)"