158 lines
6.4 KiB
Python
158 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""Collect scripts from a fixed seed repository list into this Qinglong archive."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import urllib.parse
|
|
from typing import Dict, List
|
|
|
|
import collector as c
|
|
|
|
SEEDS = [
|
|
("zy库", "https://gitee.com/jdqlscript/zy"),
|
|
("QLScriptPublic", "https://github.com/smallfawn/QLScriptPublic"),
|
|
("艾默库", "https://github.com/imoki/sign_script"),
|
|
("i茅台", "https://github.com/AkenClub/ken-iMoutai-Script"),
|
|
("偷撸库", "https://gitee.com/jdqlscript/toulu"),
|
|
("滑稽库", "https://github.com/huaji8/huajiScript"),
|
|
("其他签到库1", "https://github.com/xzxxn777/Surge"),
|
|
("其他签到库2", "https://gitee.com/jdqlscript/zyqinglong"),
|
|
("其他签到库3", "https://github.com/NaroisCool/naro-scripts"),
|
|
("其他签到库4", "https://github.com/Sitoi/dailycheckin"),
|
|
("其他签到库5", "https://github.com/sudojia/AutoTaskScript"),
|
|
("其他签到库6", "https://github.com/985Ming/qlk"),
|
|
("哔哩哔哩", "https://github.com/RayWangQvQ/BiliBiliToolPro"),
|
|
("自用青龙", "https://github.com/linbailo/zyqinglong"),
|
|
("zero205", "https://github.com/zero205/JD_tencent_scf"),
|
|
("yangro", "https://gitee.com/yangro/ql"),
|
|
("sync", "https://github.com/Aaron-lv/sync"),
|
|
("JD-Script", "https://github.com/curtinlv/JD-Script"),
|
|
("什么值得买", "https://gitee.com/jdqlscript/smzdm_script"),
|
|
("营业厅库", "https://gitee.com/jdqlscript/yyt"),
|
|
("其他签到库7", "https://github.com/xfmeng970526/ql"),
|
|
("feverrun", "https://github.com/feverrun/my_scripts"),
|
|
("WX协议本", "https://github.com/LinYuanovo/AutoTaskScripts"),
|
|
("WX协议本1", "https://github.com/3ixi/CodeScripts"),
|
|
]
|
|
|
|
MAX_PER_REPO = int(os.getenv("SEED_MAX_FILES_PER_REPO", "160"))
|
|
|
|
|
|
def parse_repo(url: str):
|
|
u = urllib.parse.urlparse(url.rstrip("/"))
|
|
parts = [p for p in u.path.split("/") if p]
|
|
if len(parts) < 2:
|
|
return None
|
|
full = "/".join(parts[:2])
|
|
if "github.com" in u.netloc.lower():
|
|
return "GitHub", full
|
|
if "gitee.com" in u.netloc.lower():
|
|
return "Gitee", full
|
|
return None
|
|
|
|
|
|
def get_github_repo(full: str) -> Dict:
|
|
r = c.github_get(f"https://api.github.com/repos/{full}")
|
|
if r:
|
|
return r.json()
|
|
return {"full_name": full, "default_branch": "main", "pushed_at": ""}
|
|
|
|
|
|
def get_gitee_repo(full: str) -> Dict:
|
|
encoded = urllib.parse.quote(full, safe="")
|
|
params = {}
|
|
if c.GITEE_TOKEN:
|
|
params["access_token"] = c.GITEE_TOKEN
|
|
r = c.generic_get(
|
|
f"https://gitee.com/api/v5/repos/{encoded}",
|
|
params=params,
|
|
headers={"Authorization": f"token {c.GITEE_TOKEN}"} if c.GITEE_TOKEN else {},
|
|
)
|
|
if r:
|
|
try:
|
|
data = r.json()
|
|
if isinstance(data, dict):
|
|
data.setdefault("full_name", full)
|
|
return data
|
|
except Exception:
|
|
pass
|
|
return {"full_name": full, "path_with_namespace": full, "default_branch": "master", "pushed_at": ""}
|
|
|
|
|
|
def sort_candidates(items: List[dict]) -> List[dict]:
|
|
candidates = [x for x in items if x.get("type") in ("blob", "file") and c.is_candidate_path(x.get("path", ""))]
|
|
candidates.sort(key=lambda x: (
|
|
0 if re.search(r"(?i)(script|scripts|ql|青龙|wx_|applet|env|bili|smzdm|yyt|moutai|茅台|营业厅|值得买)", x.get("path", "")) else 1,
|
|
1 if re.search(r"(?i)(jd_|jingdong|京东|京喜)", x.get("path", "")) else 0,
|
|
len(x.get("path", "")),
|
|
))
|
|
return candidates
|
|
|
|
|
|
def collect_seed(label: str, url: str, records: List[c.Record], seen: set, files_done: int) -> int:
|
|
parsed = parse_repo(url)
|
|
if not parsed:
|
|
print(f"[SEED_SKIP] invalid_url label={label} url={url}", flush=True)
|
|
return files_done
|
|
platform, full = parsed
|
|
print(f"\n[SEED] {label} | {platform} | {full}", flush=True)
|
|
added_before = len(records)
|
|
if platform == "GitHub":
|
|
repo = get_github_repo(full)
|
|
branch = str(repo.get("default_branch") or "main")
|
|
tree = c.list_github_tree(repo)
|
|
candidates = sort_candidates(tree)
|
|
print(f" [TREE] branch={branch} candidates={len(candidates)}", flush=True)
|
|
for item in candidates[:MAX_PER_REPO]:
|
|
path = item.get("path", "")
|
|
content, raw_url, html_url = c.get_file_content(full, path, branch)
|
|
uploaded_at = c.get_file_uploaded_at(full, path, repo.get("pushed_at", "")) if content else ""
|
|
if c.add_content_record(records, seen, content or "", platform, full, path, html_url, raw_url, uploaded_at):
|
|
files_done += 1
|
|
else:
|
|
repo = get_gitee_repo(full)
|
|
branch = str(repo.get("default_branch") or "master")
|
|
tree = c.list_gitee_tree(repo)
|
|
branch = str(repo.get("default_branch") or branch)
|
|
candidates = sort_candidates(tree)
|
|
print(f" [TREE] branch={branch} candidates={len(candidates)}", flush=True)
|
|
for item in candidates[:MAX_PER_REPO]:
|
|
path = item.get("path", "")
|
|
content, raw_url, html_url = c.get_gitee_file_content(full, path, branch)
|
|
uploaded_at = c.get_gitee_file_uploaded_at(full, path, repo.get("pushed_at", "")) if content else ""
|
|
if c.add_content_record(records, seen, content or "", platform, full, path, html_url, raw_url, uploaded_at):
|
|
files_done += 1
|
|
print(f" [SEED_DONE] added={len(records) - added_before}", flush=True)
|
|
return files_done
|
|
|
|
|
|
def main():
|
|
c.OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
c.META_DIR.mkdir(parents=True, exist_ok=True)
|
|
seen = c.load_seen_hashes()
|
|
records: List[c.Record] = []
|
|
files_done = 0
|
|
for label, url in SEEDS:
|
|
try:
|
|
files_done = collect_seed(label, url, records, seen, files_done)
|
|
except RuntimeError as e:
|
|
if str(e) == "github_rate_limited":
|
|
print("[WARN] GitHub rate limited; continuing with later non-GitHub seeds where possible", flush=True)
|
|
continue
|
|
raise
|
|
except Exception as e:
|
|
print(f"[SEED_ERROR] {label} {url}: {type(e).__name__}: {e}", flush=True)
|
|
continue
|
|
if records:
|
|
c.save_indexes(records)
|
|
c.update_readme()
|
|
else:
|
|
c.rebuild_index_from_files()
|
|
print(f"\n[DONE] seed_added={len(records)} files_done={files_done}", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|