#!/usr/bin/env bash
set -u

cd /opt/data/workspace/qinglong

START_TS=$(date +%s)
END_TS=$((START_TS + 8*60*60))
ROUND=0
LOG_DIR="logs"
mkdir -p "$LOG_DIR"
LOG_FILE="$LOG_DIR/collect_8h_$(date +%Y%m%d_%H%M%S).log"

log() {
  echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG_FILE"
}

commit_if_changed() {
  if [[ -n "$(git status --short)" ]]; then
    log "[GIT] changes detected, committing..."
    git add . >>"$LOG_FILE" 2>&1
    git commit -m "Collect Qinglong scripts batch $(date '+%Y-%m-%d %H:%M')" >>"$LOG_FILE" 2>&1 || true
    git push origin main >>"$LOG_FILE" 2>&1 || log "[GIT] push failed; will retry next round"
  else
    log "[GIT] no changes"
  fi
}

count_records() {
  python3 - <<'PY' 2>/dev/null || echo 0
import json
try:
    print(len(json.load(open('INDEX.json', encoding='utf-8'))))
except Exception:
    print(0)
PY
}

log "[START] 8h collector started"
log "[INFO] repo=$(git remote get-url origin | sed 's#oauth2:[^@]*@#oauth2:***@#')"
log "[INFO] head=$(git rev-parse --short HEAD 2>/dev/null || true)"

while [[ $(date +%s) -lt $END_TS ]]; do
  ROUND=$((ROUND + 1))
  NOW=$(date +%s)
  LEFT=$((END_TS - NOW))
  BEFORE=$(count_records)
  log "========== ROUND $ROUND START | left=${LEFT}s | records_before=${BEFORE} =========="

  # 低速采集：让 GitHub 限流后可等待更久；公开网页搜索开启；京东过滤由 collector.py 内置。
  MAX_REPOS_PER_QUERY=4 \
  MAX_FILES_TOTAL=180 \
  REQUEST_SLEEP=1.2 \
  ENABLE_WEB_SEARCH=true \
  GITHUB_RETRY_MAX_WAIT=2400 \
  uv run --with requests python -u collector.py >>"$LOG_FILE" 2>&1 || log "[WARN] collector exited non-zero; continue next round"

  # 重建索引，确保中断/限流后的落盘文件也进入索引，同时再次执行京东过滤。
  uv run --with requests python -u -c "import collector; collector.rebuild_index_from_files()" >>"$LOG_FILE" 2>&1 || log "[WARN] rebuild index failed"

  AFTER=$(count_records)
  log "[ROUND $ROUND] records_after=${AFTER} delta=$((AFTER - BEFORE))"

  # 简单扫描京东关键词，若发现则记录；collector 重建时会尽量删除命中文件。
  python3 - <<'PY' >>"$LOG_FILE" 2>&1 || true
import os
kws=['京东','jd_','JD_COOKIE','pt_key','pt_pin','api.m.jd.com','m.jd.com','jingdong','jd.com']
hits=[]
for root,_,files in os.walk('脚本库'):
    for f in files:
        p=os.path.join(root,f)
        try:
            txt=open(p,encoding='utf-8',errors='ignore').read(50000)
        except Exception:
            continue
        hay=(p+'\n'+txt).lower()
        if any(k.lower() in hay for k in kws):
            hits.append(p)
print('[JD_SCAN] hits=', len(hits))
for h in hits[:20]: print('[JD_SCAN_HIT]', h)
PY

  commit_if_changed

  NOW=$(date +%s)
  if [[ $NOW -ge $END_TS ]]; then
    break
  fi

  # 慢速循环：每轮后休眠 12-20 分钟，避免搜索端限流；剩余时间不足则按剩余时间睡。
  SLEEP=$((720 + RANDOM % 480))
  LEFT=$((END_TS - NOW))
  if [[ $SLEEP -gt $LEFT ]]; then SLEEP=$LEFT; fi
  log "[SLEEP] ${SLEEP}s"
  sleep "$SLEEP"
done

log "[FINAL] rebuilding index and pushing final state"
uv run --with requests python -u -c "import collector; collector.rebuild_index_from_files()" >>"$LOG_FILE" 2>&1 || true
commit_if_changed
log "[DONE] 8h collector finished | records=$(count_records) | head=$(git rev-parse --short HEAD 2>/dev/null || true)"