101 lines
3.3 KiB
Bash
Executable File
101 lines
3.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -u
|
||
|
||
cd /opt/data/workspace/qinglong
|
||
|
||
START_TS=$(date +%s)
|
||
END_TS=$((START_TS + 8*60*60))
|
||
ROUND=0
|
||
LOG_DIR="logs"
|
||
mkdir -p "$LOG_DIR"
|
||
LOG_FILE="$LOG_DIR/collect_8h_$(date +%Y%m%d_%H%M%S).log"
|
||
|
||
log() {
|
||
echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG_FILE"
|
||
}
|
||
|
||
commit_if_changed() {
|
||
if [[ -n "$(git status --short)" ]]; then
|
||
log "[GIT] changes detected, committing..."
|
||
git add . >>"$LOG_FILE" 2>&1
|
||
git commit -m "Collect Qinglong scripts batch $(date '+%Y-%m-%d %H:%M')" >>"$LOG_FILE" 2>&1 || true
|
||
git push origin main >>"$LOG_FILE" 2>&1 || log "[GIT] push failed; will retry next round"
|
||
else
|
||
log "[GIT] no changes"
|
||
fi
|
||
}
|
||
|
||
count_records() {
|
||
python3 - <<'PY' 2>/dev/null || echo 0
|
||
import json
|
||
try:
|
||
print(len(json.load(open('INDEX.json', encoding='utf-8'))))
|
||
except Exception:
|
||
print(0)
|
||
PY
|
||
}
|
||
|
||
log "[START] 8h collector started"
|
||
log "[INFO] repo=$(git remote get-url origin | sed 's#oauth2:[^@]*@#oauth2:***@#')"
|
||
log "[INFO] head=$(git rev-parse --short HEAD 2>/dev/null || true)"
|
||
|
||
while [[ $(date +%s) -lt $END_TS ]]; do
|
||
ROUND=$((ROUND + 1))
|
||
NOW=$(date +%s)
|
||
LEFT=$((END_TS - NOW))
|
||
BEFORE=$(count_records)
|
||
log "========== ROUND $ROUND START | left=${LEFT}s | records_before=${BEFORE} =========="
|
||
|
||
# 低速采集：让 GitHub 限流后可等待更久；公开网页搜索开启；京东过滤由 collector.py 内置。
|
||
MAX_REPOS_PER_QUERY=4 \
|
||
MAX_FILES_TOTAL=180 \
|
||
REQUEST_SLEEP=1.2 \
|
||
ENABLE_WEB_SEARCH=true \
|
||
GITHUB_RETRY_MAX_WAIT=2400 \
|
||
uv run --with requests python -u collector.py >>"$LOG_FILE" 2>&1 || log "[WARN] collector exited non-zero; continue next round"
|
||
|
||
# 重建索引，确保中断/限流后的落盘文件也进入索引，同时再次执行京东过滤。
|
||
uv run --with requests python -u -c "import collector; collector.rebuild_index_from_files()" >>"$LOG_FILE" 2>&1 || log "[WARN] rebuild index failed"
|
||
|
||
AFTER=$(count_records)
|
||
log "[ROUND $ROUND] records_after=${AFTER} delta=$((AFTER - BEFORE))"
|
||
|
||
# 简单扫描京东关键词，若发现则记录；collector 重建时会尽量删除命中文件。
|
||
python3 - <<'PY' >>"$LOG_FILE" 2>&1 || true
|
||
import os
|
||
kws=['京东','jd_','JD_COOKIE','pt_key','pt_pin','api.m.jd.com','m.jd.com','jingdong','jd.com']
|
||
hits=[]
|
||
for root,_,files in os.walk('脚本库'):
|
||
for f in files:
|
||
p=os.path.join(root,f)
|
||
try:
|
||
txt=open(p,encoding='utf-8',errors='ignore').read(50000)
|
||
except Exception:
|
||
continue
|
||
hay=(p+'\n'+txt).lower()
|
||
if any(k.lower() in hay for k in kws):
|
||
hits.append(p)
|
||
print('[JD_SCAN] hits=', len(hits))
|
||
for h in hits[:20]: print('[JD_SCAN_HIT]', h)
|
||
PY
|
||
|
||
commit_if_changed
|
||
|
||
NOW=$(date +%s)
|
||
if [[ $NOW -ge $END_TS ]]; then
|
||
break
|
||
fi
|
||
|
||
# 慢速循环：每轮后休眠 12-20 分钟，避免搜索端限流；剩余时间不足则按剩余时间睡。
|
||
SLEEP=$((720 + RANDOM % 480))
|
||
LEFT=$((END_TS - NOW))
|
||
if [[ $SLEEP -gt $LEFT ]]; then SLEEP=$LEFT; fi
|
||
log "[SLEEP] ${SLEEP}s"
|
||
sleep "$SLEEP"
|
||
done
|
||
|
||
log "[FINAL] rebuilding index and pushing final state"
|
||
uv run --with requests python -u -c "import collector; collector.rebuild_index_from_files()" >>"$LOG_FILE" 2>&1 || true
|
||
commit_if_changed
|
||
log "[DONE] 8h collector finished | records=$(count_records) | head=$(git rev-parse --short HEAD 2>/dev/null || true)"
|