Collect seeded Qinglong script repositories
This commit is contained in:
Hermes Agent
181
seed_collect_git.py
Normal file
181
seed_collect_git.py
Normal file
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Clone fixed seed repositories and collect Qinglong scripts without GitHub/Gitee API rate limits."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
import collector as c
|
||||
|
||||
SEEDS: List[Tuple[str, str]] = [
|
||||
("zy库", "https://gitee.com/jdqlscript/zy"),
|
||||
("QLScriptPublic", "https://github.com/smallfawn/QLScriptPublic"),
|
||||
("艾默库", "https://github.com/imoki/sign_script"),
|
||||
("i茅台", "https://github.com/AkenClub/ken-iMoutai-Script"),
|
||||
("偷撸库", "https://gitee.com/jdqlscript/toulu"),
|
||||
("滑稽库", "https://github.com/huaji8/huajiScript"),
|
||||
("其他签到库1", "https://github.com/xzxxn777/Surge"),
|
||||
("其他签到库2", "https://gitee.com/jdqlscript/zyqinglong"),
|
||||
("其他签到库3", "https://github.com/NaroisCool/naro-scripts"),
|
||||
("其他签到库4", "https://github.com/Sitoi/dailycheckin"),
|
||||
("其他签到库5", "https://github.com/sudojia/AutoTaskScript"),
|
||||
("其他签到库6", "https://github.com/985Ming/qlk"),
|
||||
("哔哩哔哩", "https://github.com/RayWangQvQ/BiliBiliToolPro"),
|
||||
("自用青龙", "https://github.com/linbailo/zyqinglong"),
|
||||
("zero205", "https://github.com/zero205/JD_tencent_scf"),
|
||||
("yangro", "https://gitee.com/yangro/ql"),
|
||||
("sync", "https://github.com/Aaron-lv/sync"),
|
||||
("JD-Script", "https://github.com/curtinlv/JD-Script"),
|
||||
("什么值得买", "https://gitee.com/jdqlscript/smzdm_script"),
|
||||
("营业厅库", "https://gitee.com/jdqlscript/yyt"),
|
||||
("其他签到库7", "https://github.com/xfmeng970526/ql"),
|
||||
("feverrun", "https://github.com/feverrun/my_scripts"),
|
||||
("WX协议本", "https://github.com/LinYuanovo/AutoTaskScripts"),
|
||||
("WX协议本1", "https://github.com/3ixi/CodeScripts"),
|
||||
]
|
||||
|
||||
CACHE_ROOT = c.META_DIR / "seed_repos"
|
||||
MAX_PER_REPO = int(os.getenv("SEED_MAX_FILES_PER_REPO", "260"))
|
||||
GIT_TIMEOUT = int(os.getenv("SEED_GIT_TIMEOUT", "180"))
|
||||
SOCKS_PROXY = os.getenv("SOCKS_PROXY", "socks5://192.168.0.182:1080")
|
||||
|
||||
|
||||
def safe_dir(url: str) -> str:
|
||||
u = urllib.parse.urlparse(url.rstrip("/"))
|
||||
parts = [p for p in u.path.split("/") if p]
|
||||
return c.safe_segment((u.netloc + "_" + "_".join(parts[:2])).replace(".", "_"), max_len=120)
|
||||
|
||||
|
||||
def repo_full(url: str) -> str:
|
||||
u = urllib.parse.urlparse(url.rstrip("/"))
|
||||
parts = [p for p in u.path.split("/") if p]
|
||||
return "/".join(parts[:2]) if len(parts) >= 2 else url
|
||||
|
||||
|
||||
def platform(url: str) -> str:
|
||||
return "Gitee" if "gitee.com" in urllib.parse.urlparse(url).netloc.lower() else "GitHub"
|
||||
|
||||
|
||||
def run(cmd: List[str], cwd: Path | None = None, use_proxy: bool = False) -> subprocess.CompletedProcess:
|
||||
env = os.environ.copy()
|
||||
if use_proxy:
|
||||
env.setdefault("ALL_PROXY", SOCKS_PROXY)
|
||||
env.setdefault("HTTPS_PROXY", SOCKS_PROXY)
|
||||
env.setdefault("HTTP_PROXY", SOCKS_PROXY)
|
||||
return subprocess.run(cmd, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=GIT_TIMEOUT)
|
||||
|
||||
|
||||
def clone_or_update(label: str, url: str) -> Path | None:
|
||||
CACHE_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
dst = CACHE_ROOT / safe_dir(url)
|
||||
if (dst / ".git").exists():
|
||||
print(f" [GIT] update {dst.name}", flush=True)
|
||||
r = run(["git", "fetch", "--depth", "1", "origin"], cwd=dst)
|
||||
if r.returncode != 0:
|
||||
print(f" [GIT_WARN] fetch failed: {r.stdout[-400:]}", flush=True)
|
||||
r = run(["git", "reset", "--hard", "FETCH_HEAD"], cwd=dst)
|
||||
if r.returncode == 0:
|
||||
return dst
|
||||
if dst.exists():
|
||||
shutil.rmtree(dst, ignore_errors=True)
|
||||
print(f" [GIT] clone {url}", flush=True)
|
||||
r = run(["git", "clone", "--depth", "1", url, str(dst)])
|
||||
if r.returncode != 0 and "github.com" in url:
|
||||
print(" [GIT_WARN] direct clone failed, retry with SOCKS proxy", flush=True)
|
||||
r = run(["git", "clone", "--depth", "1", url, str(dst)], use_proxy=True)
|
||||
if r.returncode != 0:
|
||||
print(f" [GIT_FAIL] {label}: {r.stdout[-800:]}", flush=True)
|
||||
return None
|
||||
return dst
|
||||
|
||||
|
||||
def rel_url(url: str, branch: str, rel: str, raw: bool) -> str:
|
||||
full = repo_full(url)
|
||||
qrel = "/".join(urllib.parse.quote(p) for p in rel.split("/"))
|
||||
if platform(url) == "GitHub":
|
||||
return f"https://raw.githubusercontent.com/{full}/{branch}/{qrel}" if raw else f"https://github.com/{full}/blob/{branch}/{qrel}"
|
||||
return f"https://gitee.com/{full}/raw/{branch}/{qrel}" if raw else f"https://gitee.com/{full}/blob/{branch}/{qrel}"
|
||||
|
||||
|
||||
def default_branch(repo: Path) -> str:
|
||||
for cmd in (["git", "symbolic-ref", "--short", "HEAD"], ["git", "rev-parse", "--abbrev-ref", "HEAD"]):
|
||||
r = run(cmd, cwd=repo)
|
||||
if r.returncode == 0 and r.stdout.strip():
|
||||
return r.stdout.strip()
|
||||
return "main"
|
||||
|
||||
|
||||
def last_commit_date(repo: Path, rel: str) -> str:
|
||||
r = run(["git", "log", "-1", "--format=%cI", "--", rel], cwd=repo)
|
||||
return r.stdout.strip() if r.returncode == 0 else ""
|
||||
|
||||
|
||||
def iter_candidates(repo: Path):
|
||||
files = []
|
||||
for p in repo.rglob("*"):
|
||||
if not p.is_file():
|
||||
continue
|
||||
rel = p.relative_to(repo).as_posix()
|
||||
if rel.startswith(".git/") or not c.is_candidate_path(rel):
|
||||
continue
|
||||
files.append(rel)
|
||||
files.sort(key=lambda path: (
|
||||
0 if re.search(r"(?i)(script|scripts|ql|青龙|wx_|applet|env|bili|smzdm|yyt|moutai|茅台|营业厅|值得买)", path) else 1,
|
||||
1 if re.search(r"(?i)(jd_|jingdong|京东|京喜)", path) else 0,
|
||||
len(path),
|
||||
))
|
||||
return files[:MAX_PER_REPO]
|
||||
|
||||
|
||||
def collect_repo(label: str, url: str, records: List[c.Record], seen: set):
|
||||
print(f"\n[SEED_CLONE] {label} | {url}", flush=True)
|
||||
repo = clone_or_update(label, url)
|
||||
if not repo:
|
||||
return
|
||||
full = repo_full(url)
|
||||
br = default_branch(repo)
|
||||
before = len(records)
|
||||
candidates = iter_candidates(repo)
|
||||
print(f" [SCAN] branch={br} candidates={len(candidates)}", flush=True)
|
||||
for rel in candidates:
|
||||
p = repo / rel
|
||||
try:
|
||||
if p.stat().st_size > 1024 * 1024:
|
||||
continue
|
||||
content = p.read_text(encoding="utf-8", errors="ignore")
|
||||
except Exception:
|
||||
continue
|
||||
uploaded = last_commit_date(repo, rel)
|
||||
c.add_content_record(
|
||||
records, seen, content, platform(url), full, rel,
|
||||
rel_url(url, br, rel, raw=False), rel_url(url, br, rel, raw=True), uploaded,
|
||||
)
|
||||
print(f" [SEED_DONE] added={len(records)-before}", flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
c.OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
c.META_DIR.mkdir(parents=True, exist_ok=True)
|
||||
seen = c.load_seen_hashes()
|
||||
records: List[c.Record] = []
|
||||
for label, url in SEEDS:
|
||||
try:
|
||||
collect_repo(label, url, records, seen)
|
||||
except Exception as e:
|
||||
print(f"[SEED_ERROR] {label}: {type(e).__name__}: {e}", flush=True)
|
||||
if records:
|
||||
c.save_indexes(records)
|
||||
c.update_readme()
|
||||
else:
|
||||
c.rebuild_index_from_files()
|
||||
print(f"\n[DONE] clone_seed_added={len(records)}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user