182 lines
7.1 KiB
Python
182 lines
7.1 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""Clone fixed seed repositories and collect Qinglong scripts without GitHub/Gitee API rate limits."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
from typing import List, Tuple
|
|
|
|
import collector as c
|
|
|
|
SEEDS: List[Tuple[str, str]] = [
|
|
("zy库", "https://gitee.com/jdqlscript/zy"),
|
|
("QLScriptPublic", "https://github.com/smallfawn/QLScriptPublic"),
|
|
("艾默库", "https://github.com/imoki/sign_script"),
|
|
("i茅台", "https://github.com/AkenClub/ken-iMoutai-Script"),
|
|
("偷撸库", "https://gitee.com/jdqlscript/toulu"),
|
|
("滑稽库", "https://github.com/huaji8/huajiScript"),
|
|
("其他签到库1", "https://github.com/xzxxn777/Surge"),
|
|
("其他签到库2", "https://gitee.com/jdqlscript/zyqinglong"),
|
|
("其他签到库3", "https://github.com/NaroisCool/naro-scripts"),
|
|
("其他签到库4", "https://github.com/Sitoi/dailycheckin"),
|
|
("其他签到库5", "https://github.com/sudojia/AutoTaskScript"),
|
|
("其他签到库6", "https://github.com/985Ming/qlk"),
|
|
("哔哩哔哩", "https://github.com/RayWangQvQ/BiliBiliToolPro"),
|
|
("自用青龙", "https://github.com/linbailo/zyqinglong"),
|
|
("zero205", "https://github.com/zero205/JD_tencent_scf"),
|
|
("yangro", "https://gitee.com/yangro/ql"),
|
|
("sync", "https://github.com/Aaron-lv/sync"),
|
|
("JD-Script", "https://github.com/curtinlv/JD-Script"),
|
|
("什么值得买", "https://gitee.com/jdqlscript/smzdm_script"),
|
|
("营业厅库", "https://gitee.com/jdqlscript/yyt"),
|
|
("其他签到库7", "https://github.com/xfmeng970526/ql"),
|
|
("feverrun", "https://github.com/feverrun/my_scripts"),
|
|
("WX协议本", "https://github.com/LinYuanovo/AutoTaskScripts"),
|
|
("WX协议本1", "https://github.com/3ixi/CodeScripts"),
|
|
]
|
|
|
|
CACHE_ROOT = c.META_DIR / "seed_repos"
|
|
MAX_PER_REPO = int(os.getenv("SEED_MAX_FILES_PER_REPO", "260"))
|
|
GIT_TIMEOUT = int(os.getenv("SEED_GIT_TIMEOUT", "180"))
|
|
SOCKS_PROXY = os.getenv("SOCKS_PROXY", "socks5://192.168.0.182:1080")
|
|
|
|
|
|
def safe_dir(url: str) -> str:
|
|
u = urllib.parse.urlparse(url.rstrip("/"))
|
|
parts = [p for p in u.path.split("/") if p]
|
|
return c.safe_segment((u.netloc + "_" + "_".join(parts[:2])).replace(".", "_"), max_len=120)
|
|
|
|
|
|
def repo_full(url: str) -> str:
|
|
u = urllib.parse.urlparse(url.rstrip("/"))
|
|
parts = [p for p in u.path.split("/") if p]
|
|
return "/".join(parts[:2]) if len(parts) >= 2 else url
|
|
|
|
|
|
def platform(url: str) -> str:
|
|
return "Gitee" if "gitee.com" in urllib.parse.urlparse(url).netloc.lower() else "GitHub"
|
|
|
|
|
|
def run(cmd: List[str], cwd: Path | None = None, use_proxy: bool = False) -> subprocess.CompletedProcess:
|
|
env = os.environ.copy()
|
|
if use_proxy:
|
|
env.setdefault("ALL_PROXY", SOCKS_PROXY)
|
|
env.setdefault("HTTPS_PROXY", SOCKS_PROXY)
|
|
env.setdefault("HTTP_PROXY", SOCKS_PROXY)
|
|
return subprocess.run(cmd, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=GIT_TIMEOUT)
|
|
|
|
|
|
def clone_or_update(label: str, url: str) -> Path | None:
|
|
CACHE_ROOT.mkdir(parents=True, exist_ok=True)
|
|
dst = CACHE_ROOT / safe_dir(url)
|
|
if (dst / ".git").exists():
|
|
print(f" [GIT] update {dst.name}", flush=True)
|
|
r = run(["git", "fetch", "--depth", "1", "origin"], cwd=dst)
|
|
if r.returncode != 0:
|
|
print(f" [GIT_WARN] fetch failed: {r.stdout[-400:]}", flush=True)
|
|
r = run(["git", "reset", "--hard", "FETCH_HEAD"], cwd=dst)
|
|
if r.returncode == 0:
|
|
return dst
|
|
if dst.exists():
|
|
shutil.rmtree(dst, ignore_errors=True)
|
|
print(f" [GIT] clone {url}", flush=True)
|
|
r = run(["git", "clone", "--depth", "1", url, str(dst)])
|
|
if r.returncode != 0 and "github.com" in url:
|
|
print(" [GIT_WARN] direct clone failed, retry with SOCKS proxy", flush=True)
|
|
r = run(["git", "clone", "--depth", "1", url, str(dst)], use_proxy=True)
|
|
if r.returncode != 0:
|
|
print(f" [GIT_FAIL] {label}: {r.stdout[-800:]}", flush=True)
|
|
return None
|
|
return dst
|
|
|
|
|
|
def rel_url(url: str, branch: str, rel: str, raw: bool) -> str:
|
|
full = repo_full(url)
|
|
qrel = "/".join(urllib.parse.quote(p) for p in rel.split("/"))
|
|
if platform(url) == "GitHub":
|
|
return f"https://raw.githubusercontent.com/{full}/{branch}/{qrel}" if raw else f"https://github.com/{full}/blob/{branch}/{qrel}"
|
|
return f"https://gitee.com/{full}/raw/{branch}/{qrel}" if raw else f"https://gitee.com/{full}/blob/{branch}/{qrel}"
|
|
|
|
|
|
def default_branch(repo: Path) -> str:
|
|
for cmd in (["git", "symbolic-ref", "--short", "HEAD"], ["git", "rev-parse", "--abbrev-ref", "HEAD"]):
|
|
r = run(cmd, cwd=repo)
|
|
if r.returncode == 0 and r.stdout.strip():
|
|
return r.stdout.strip()
|
|
return "main"
|
|
|
|
|
|
def last_commit_date(repo: Path, rel: str) -> str:
|
|
r = run(["git", "log", "-1", "--format=%cI", "--", rel], cwd=repo)
|
|
return r.stdout.strip() if r.returncode == 0 else ""
|
|
|
|
|
|
def iter_candidates(repo: Path):
|
|
files = []
|
|
for p in repo.rglob("*"):
|
|
if not p.is_file():
|
|
continue
|
|
rel = p.relative_to(repo).as_posix()
|
|
if rel.startswith(".git/") or not c.is_candidate_path(rel):
|
|
continue
|
|
files.append(rel)
|
|
files.sort(key=lambda path: (
|
|
0 if re.search(r"(?i)(script|scripts|ql|青龙|wx_|applet|env|bili|smzdm|yyt|moutai|茅台|营业厅|值得买)", path) else 1,
|
|
1 if re.search(r"(?i)(jd_|jingdong|京东|京喜)", path) else 0,
|
|
len(path),
|
|
))
|
|
return files[:MAX_PER_REPO]
|
|
|
|
|
|
def collect_repo(label: str, url: str, records: List[c.Record], seen: set):
|
|
print(f"\n[SEED_CLONE] {label} | {url}", flush=True)
|
|
repo = clone_or_update(label, url)
|
|
if not repo:
|
|
return
|
|
full = repo_full(url)
|
|
br = default_branch(repo)
|
|
before = len(records)
|
|
candidates = iter_candidates(repo)
|
|
print(f" [SCAN] branch={br} candidates={len(candidates)}", flush=True)
|
|
for rel in candidates:
|
|
p = repo / rel
|
|
try:
|
|
if p.stat().st_size > 1024 * 1024:
|
|
continue
|
|
content = p.read_text(encoding="utf-8", errors="ignore")
|
|
except Exception:
|
|
continue
|
|
uploaded = last_commit_date(repo, rel)
|
|
c.add_content_record(
|
|
records, seen, content, platform(url), full, rel,
|
|
rel_url(url, br, rel, raw=False), rel_url(url, br, rel, raw=True), uploaded,
|
|
)
|
|
print(f" [SEED_DONE] added={len(records)-before}", flush=True)
|
|
|
|
|
|
def main():
|
|
c.OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
c.META_DIR.mkdir(parents=True, exist_ok=True)
|
|
seen = c.load_seen_hashes()
|
|
records: List[c.Record] = []
|
|
for label, url in SEEDS:
|
|
try:
|
|
collect_repo(label, url, records, seen)
|
|
except Exception as e:
|
|
print(f"[SEED_ERROR] {label}: {type(e).__name__}: {e}", flush=True)
|
|
if records:
|
|
c.save_indexes(records)
|
|
c.update_readme()
|
|
else:
|
|
c.rebuild_index_from_files()
|
|
print(f"\n[DONE] clone_seed_added={len(records)}", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|