Collect seeded Qinglong script repositories
This commit is contained in:
Hermes Agent
157
seed_collect.py
Normal file
157
seed_collect.py
Normal file
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Collect scripts from a fixed seed repository list into this Qinglong archive."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import urllib.parse
|
||||
from typing import Dict, List
|
||||
|
||||
import collector as c
|
||||
|
||||
SEEDS = [
|
||||
("zy库", "https://gitee.com/jdqlscript/zy"),
|
||||
("QLScriptPublic", "https://github.com/smallfawn/QLScriptPublic"),
|
||||
("艾默库", "https://github.com/imoki/sign_script"),
|
||||
("i茅台", "https://github.com/AkenClub/ken-iMoutai-Script"),
|
||||
("偷撸库", "https://gitee.com/jdqlscript/toulu"),
|
||||
("滑稽库", "https://github.com/huaji8/huajiScript"),
|
||||
("其他签到库1", "https://github.com/xzxxn777/Surge"),
|
||||
("其他签到库2", "https://gitee.com/jdqlscript/zyqinglong"),
|
||||
("其他签到库3", "https://github.com/NaroisCool/naro-scripts"),
|
||||
("其他签到库4", "https://github.com/Sitoi/dailycheckin"),
|
||||
("其他签到库5", "https://github.com/sudojia/AutoTaskScript"),
|
||||
("其他签到库6", "https://github.com/985Ming/qlk"),
|
||||
("哔哩哔哩", "https://github.com/RayWangQvQ/BiliBiliToolPro"),
|
||||
("自用青龙", "https://github.com/linbailo/zyqinglong"),
|
||||
("zero205", "https://github.com/zero205/JD_tencent_scf"),
|
||||
("yangro", "https://gitee.com/yangro/ql"),
|
||||
("sync", "https://github.com/Aaron-lv/sync"),
|
||||
("JD-Script", "https://github.com/curtinlv/JD-Script"),
|
||||
("什么值得买", "https://gitee.com/jdqlscript/smzdm_script"),
|
||||
("营业厅库", "https://gitee.com/jdqlscript/yyt"),
|
||||
("其他签到库7", "https://github.com/xfmeng970526/ql"),
|
||||
("feverrun", "https://github.com/feverrun/my_scripts"),
|
||||
("WX协议本", "https://github.com/LinYuanovo/AutoTaskScripts"),
|
||||
("WX协议本1", "https://github.com/3ixi/CodeScripts"),
|
||||
]
|
||||
|
||||
MAX_PER_REPO = int(os.getenv("SEED_MAX_FILES_PER_REPO", "160"))
|
||||
|
||||
|
||||
def parse_repo(url: str):
|
||||
u = urllib.parse.urlparse(url.rstrip("/"))
|
||||
parts = [p for p in u.path.split("/") if p]
|
||||
if len(parts) < 2:
|
||||
return None
|
||||
full = "/".join(parts[:2])
|
||||
if "github.com" in u.netloc.lower():
|
||||
return "GitHub", full
|
||||
if "gitee.com" in u.netloc.lower():
|
||||
return "Gitee", full
|
||||
return None
|
||||
|
||||
|
||||
def get_github_repo(full: str) -> Dict:
|
||||
r = c.github_get(f"https://api.github.com/repos/{full}")
|
||||
if r:
|
||||
return r.json()
|
||||
return {"full_name": full, "default_branch": "main", "pushed_at": ""}
|
||||
|
||||
|
||||
def get_gitee_repo(full: str) -> Dict:
|
||||
encoded = urllib.parse.quote(full, safe="")
|
||||
params = {}
|
||||
if c.GITEE_TOKEN:
|
||||
params["access_token"] = c.GITEE_TOKEN
|
||||
r = c.generic_get(
|
||||
f"https://gitee.com/api/v5/repos/{encoded}",
|
||||
params=params,
|
||||
headers={"Authorization": f"token {c.GITEE_TOKEN}"} if c.GITEE_TOKEN else {},
|
||||
)
|
||||
if r:
|
||||
try:
|
||||
data = r.json()
|
||||
if isinstance(data, dict):
|
||||
data.setdefault("full_name", full)
|
||||
return data
|
||||
except Exception:
|
||||
pass
|
||||
return {"full_name": full, "path_with_namespace": full, "default_branch": "master", "pushed_at": ""}
|
||||
|
||||
|
||||
def sort_candidates(items: List[dict]) -> List[dict]:
|
||||
candidates = [x for x in items if x.get("type") in ("blob", "file") and c.is_candidate_path(x.get("path", ""))]
|
||||
candidates.sort(key=lambda x: (
|
||||
0 if re.search(r"(?i)(script|scripts|ql|青龙|wx_|applet|env|bili|smzdm|yyt|moutai|茅台|营业厅|值得买)", x.get("path", "")) else 1,
|
||||
1 if re.search(r"(?i)(jd_|jingdong|京东|京喜)", x.get("path", "")) else 0,
|
||||
len(x.get("path", "")),
|
||||
))
|
||||
return candidates
|
||||
|
||||
|
||||
def collect_seed(label: str, url: str, records: List[c.Record], seen: set, files_done: int) -> int:
|
||||
parsed = parse_repo(url)
|
||||
if not parsed:
|
||||
print(f"[SEED_SKIP] invalid_url label={label} url={url}", flush=True)
|
||||
return files_done
|
||||
platform, full = parsed
|
||||
print(f"\n[SEED] {label} | {platform} | {full}", flush=True)
|
||||
added_before = len(records)
|
||||
if platform == "GitHub":
|
||||
repo = get_github_repo(full)
|
||||
branch = str(repo.get("default_branch") or "main")
|
||||
tree = c.list_github_tree(repo)
|
||||
candidates = sort_candidates(tree)
|
||||
print(f" [TREE] branch={branch} candidates={len(candidates)}", flush=True)
|
||||
for item in candidates[:MAX_PER_REPO]:
|
||||
path = item.get("path", "")
|
||||
content, raw_url, html_url = c.get_file_content(full, path, branch)
|
||||
uploaded_at = c.get_file_uploaded_at(full, path, repo.get("pushed_at", "")) if content else ""
|
||||
if c.add_content_record(records, seen, content or "", platform, full, path, html_url, raw_url, uploaded_at):
|
||||
files_done += 1
|
||||
else:
|
||||
repo = get_gitee_repo(full)
|
||||
branch = str(repo.get("default_branch") or "master")
|
||||
tree = c.list_gitee_tree(repo)
|
||||
branch = str(repo.get("default_branch") or branch)
|
||||
candidates = sort_candidates(tree)
|
||||
print(f" [TREE] branch={branch} candidates={len(candidates)}", flush=True)
|
||||
for item in candidates[:MAX_PER_REPO]:
|
||||
path = item.get("path", "")
|
||||
content, raw_url, html_url = c.get_gitee_file_content(full, path, branch)
|
||||
uploaded_at = c.get_gitee_file_uploaded_at(full, path, repo.get("pushed_at", "")) if content else ""
|
||||
if c.add_content_record(records, seen, content or "", platform, full, path, html_url, raw_url, uploaded_at):
|
||||
files_done += 1
|
||||
print(f" [SEED_DONE] added={len(records) - added_before}", flush=True)
|
||||
return files_done
|
||||
|
||||
|
||||
def main():
|
||||
c.OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
c.META_DIR.mkdir(parents=True, exist_ok=True)
|
||||
seen = c.load_seen_hashes()
|
||||
records: List[c.Record] = []
|
||||
files_done = 0
|
||||
for label, url in SEEDS:
|
||||
try:
|
||||
files_done = collect_seed(label, url, records, seen, files_done)
|
||||
except RuntimeError as e:
|
||||
if str(e) == "github_rate_limited":
|
||||
print("[WARN] GitHub rate limited; continuing with later non-GitHub seeds where possible", flush=True)
|
||||
continue
|
||||
raise
|
||||
except Exception as e:
|
||||
print(f"[SEED_ERROR] {label} {url}: {type(e).__name__}: {e}", flush=True)
|
||||
continue
|
||||
if records:
|
||||
c.save_indexes(records)
|
||||
c.update_readme()
|
||||
else:
|
||||
c.rebuild_index_from_files()
|
||||
print(f"\n[DONE] seed_added={len(records)} files_done={files_done}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user