#!/usr/bin/env python3 # -*- coding: utf-8 -*- """Clone fixed seed repositories and collect Qinglong scripts without GitHub/Gitee API rate limits.""" from __future__ import annotations import os import re import shutil import subprocess import urllib.parse from pathlib import Path from typing import List, Tuple import collector as c SEEDS: List[Tuple[str, str]] = [ ("zy库", "https://gitee.com/jdqlscript/zy"), ("QLScriptPublic", "https://github.com/smallfawn/QLScriptPublic"), ("艾默库", "https://github.com/imoki/sign_script"), ("i茅台", "https://github.com/AkenClub/ken-iMoutai-Script"), ("偷撸库", "https://gitee.com/jdqlscript/toulu"), ("滑稽库", "https://github.com/huaji8/huajiScript"), ("其他签到库1", "https://github.com/xzxxn777/Surge"), ("其他签到库2", "https://gitee.com/jdqlscript/zyqinglong"), ("其他签到库3", "https://github.com/NaroisCool/naro-scripts"), ("其他签到库4", "https://github.com/Sitoi/dailycheckin"), ("其他签到库5", "https://github.com/sudojia/AutoTaskScript"), ("其他签到库6", "https://github.com/985Ming/qlk"), ("哔哩哔哩", "https://github.com/RayWangQvQ/BiliBiliToolPro"), ("自用青龙", "https://github.com/linbailo/zyqinglong"), ("zero205", "https://github.com/zero205/JD_tencent_scf"), ("yangro", "https://gitee.com/yangro/ql"), ("sync", "https://github.com/Aaron-lv/sync"), ("JD-Script", "https://github.com/curtinlv/JD-Script"), ("什么值得买", "https://gitee.com/jdqlscript/smzdm_script"), ("营业厅库", "https://gitee.com/jdqlscript/yyt"), ("其他签到库7", "https://github.com/xfmeng970526/ql"), ("feverrun", "https://github.com/feverrun/my_scripts"), ("WX协议本", "https://github.com/LinYuanovo/AutoTaskScripts"), ("WX协议本1", "https://github.com/3ixi/CodeScripts"), ] CACHE_ROOT = c.META_DIR / "seed_repos" MAX_PER_REPO = int(os.getenv("SEED_MAX_FILES_PER_REPO", "260")) GIT_TIMEOUT = int(os.getenv("SEED_GIT_TIMEOUT", "180")) SOCKS_PROXY = os.getenv("SOCKS_PROXY", "socks5://192.168.0.182:1080") def safe_dir(url: str) -> str: u = urllib.parse.urlparse(url.rstrip("/")) parts = [p for p in u.path.split("/") if p] return c.safe_segment((u.netloc + "_" + "_".join(parts[:2])).replace(".", "_"), max_len=120) def repo_full(url: str) -> str: u = urllib.parse.urlparse(url.rstrip("/")) parts = [p for p in u.path.split("/") if p] return "/".join(parts[:2]) if len(parts) >= 2 else url def platform(url: str) -> str: return "Gitee" if "gitee.com" in urllib.parse.urlparse(url).netloc.lower() else "GitHub" def run(cmd: List[str], cwd: Path | None = None, use_proxy: bool = False) -> subprocess.CompletedProcess: env = os.environ.copy() if use_proxy: env.setdefault("ALL_PROXY", SOCKS_PROXY) env.setdefault("HTTPS_PROXY", SOCKS_PROXY) env.setdefault("HTTP_PROXY", SOCKS_PROXY) return subprocess.run(cmd, cwd=str(cwd) if cwd else None, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=GIT_TIMEOUT) def clone_or_update(label: str, url: str) -> Path | None: CACHE_ROOT.mkdir(parents=True, exist_ok=True) dst = CACHE_ROOT / safe_dir(url) if (dst / ".git").exists(): print(f" [GIT] update {dst.name}", flush=True) r = run(["git", "fetch", "--depth", "1", "origin"], cwd=dst) if r.returncode != 0: print(f" [GIT_WARN] fetch failed: {r.stdout[-400:]}", flush=True) r = run(["git", "reset", "--hard", "FETCH_HEAD"], cwd=dst) if r.returncode == 0: return dst if dst.exists(): shutil.rmtree(dst, ignore_errors=True) print(f" [GIT] clone {url}", flush=True) r = run(["git", "clone", "--depth", "1", url, str(dst)]) if r.returncode != 0 and "github.com" in url: print(" [GIT_WARN] direct clone failed, retry with SOCKS proxy", flush=True) r = run(["git", "clone", "--depth", "1", url, str(dst)], use_proxy=True) if r.returncode != 0: print(f" [GIT_FAIL] {label}: {r.stdout[-800:]}", flush=True) return None return dst def rel_url(url: str, branch: str, rel: str, raw: bool) -> str: full = repo_full(url) qrel = "/".join(urllib.parse.quote(p) for p in rel.split("/")) if platform(url) == "GitHub": return f"https://raw.githubusercontent.com/{full}/{branch}/{qrel}" if raw else f"https://github.com/{full}/blob/{branch}/{qrel}" return f"https://gitee.com/{full}/raw/{branch}/{qrel}" if raw else f"https://gitee.com/{full}/blob/{branch}/{qrel}" def default_branch(repo: Path) -> str: for cmd in (["git", "symbolic-ref", "--short", "HEAD"], ["git", "rev-parse", "--abbrev-ref", "HEAD"]): r = run(cmd, cwd=repo) if r.returncode == 0 and r.stdout.strip(): return r.stdout.strip() return "main" def last_commit_date(repo: Path, rel: str) -> str: r = run(["git", "log", "-1", "--format=%cI", "--", rel], cwd=repo) return r.stdout.strip() if r.returncode == 0 else "" def iter_candidates(repo: Path): files = [] for p in repo.rglob("*"): if not p.is_file(): continue rel = p.relative_to(repo).as_posix() if rel.startswith(".git/") or not c.is_candidate_path(rel): continue files.append(rel) files.sort(key=lambda path: ( 0 if re.search(r"(?i)(script|scripts|ql|青龙|wx_|applet|env|bili|smzdm|yyt|moutai|茅台|营业厅|值得买)", path) else 1, 1 if re.search(r"(?i)(jd_|jingdong|京东|京喜)", path) else 0, len(path), )) return files[:MAX_PER_REPO] def collect_repo(label: str, url: str, records: List[c.Record], seen: set): print(f"\n[SEED_CLONE] {label} | {url}", flush=True) repo = clone_or_update(label, url) if not repo: return full = repo_full(url) br = default_branch(repo) before = len(records) candidates = iter_candidates(repo) print(f" [SCAN] branch={br} candidates={len(candidates)}", flush=True) for rel in candidates: p = repo / rel try: if p.stat().st_size > 1024 * 1024: continue content = p.read_text(encoding="utf-8", errors="ignore") except Exception: continue uploaded = last_commit_date(repo, rel) c.add_content_record( records, seen, content, platform(url), full, rel, rel_url(url, br, rel, raw=False), rel_url(url, br, rel, raw=True), uploaded, ) print(f" [SEED_DONE] added={len(records)-before}", flush=True) def main(): c.OUT_DIR.mkdir(parents=True, exist_ok=True) c.META_DIR.mkdir(parents=True, exist_ok=True) seen = c.load_seen_hashes() records: List[c.Record] = [] for label, url in SEEDS: try: collect_repo(label, url, records, seen) except Exception as e: print(f"[SEED_ERROR] {label}: {type(e).__name__}: {e}", flush=True) if records: c.save_indexes(records) c.update_readme() else: c.rebuild_index_from_files() print(f"\n[DONE] clone_seed_added={len(records)}", flush=True) if __name__ == "__main__": main()