Exclude JD related Qinglong scripts

collector.py

@@ -23,6 +23,7 @@ import json
 import os
 import re
 import time
+import html
 import unicodedata
 import urllib.parse
 from dataclasses import dataclass, asdict
@@ -50,6 +51,7 @@ MAX_FILES_TOTAL = int(os.getenv("MAX_FILES_TOTAL", "260"))
 REQUEST_SLEEP = float(os.getenv("REQUEST_SLEEP", "0.4"))
 
 GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", "").strip()
+GITHUB_RATE_LIMIT_RESET = 0
 SESSION = requests.Session()
 SESSION.headers.update({
     "User-Agent": "ql-script-classifier/1.0",
@@ -100,6 +102,12 @@ SKIP_PATH_KEYWORDS = {
     "readme", "license", "docker", "workflow", ".github", "icon", "image", "png", "jpg",
 }
 
+JD_EXCLUDE_KEYWORDS = [
+    "京东", "东东", "jd_", "JD_", "jdCookie", "JD_COOKIE", "pt_key", "pt_pin",
+    "bean", "京豆", "joy", "宠汪汪", "东东农场", "东东萌宠", "京喜", "京东到家",
+    "api.m.jd.com", "m.jd.com", "jingdong", "jdc", "jd.com"
+]
+
 CATEGORY_ORDER = [
     "APP版/抓包",
     "APP版/账密",
@@ -149,7 +157,12 @@ def github_get(url: str, **kwargs):
     time.sleep(REQUEST_SLEEP)
     r = SESSION.get(url, timeout=30, **kwargs)
     if r.status_code == 403 and "rate limit" in r.text.lower():
+        global GITHUB_RATE_LIMIT_RESET
         reset = r.headers.get("X-RateLimit-Reset")
+        try:
+            GITHUB_RATE_LIMIT_RESET = max(GITHUB_RATE_LIMIT_RESET, int(reset or 0))
+        except Exception:
+            pass
         print(f"[WARN] GitHub rate limited: {url} reset={reset}")
         raise RuntimeError("github_rate_limited")
     if r.status_code >= 400:
@@ -262,32 +275,43 @@ def get_gitee_file_uploaded_at(repo_full: str, path: str, fallback: str) -> str:
 
 
 def discover_public_code_links() -> List[dict]:
-    """通过公开搜索页发现 raw/js/py 链接。使用 DuckDuckGo HTML，无登录。"""
+    """通过公开搜索页发现 raw/js/py 链接。优先 Bing；DuckDuckGo 可选。"""
     links: Dict[str, dict] = {}
-    if os.getenv("ENABLE_WEB_SEARCH", "false").strip().lower() != "true":
+    search_urls = []
-        print("[WEB_SEARCH] disabled; set ENABLE_WEB_SEARCH=true to enable public web search")
-        return []
     for q in WEB_SEARCH_QUERIES:
-        print(f"[WEB_SEARCH] {q}")
+        search_urls.append(("Bing", "https://www.bing.com/search", {"q": q}))
-        url = "https://duckduckgo.com/html/"
+        if os.getenv("ENABLE_DDG_SEARCH", "false").strip().lower() == "true":
-        r = generic_get(url, params={"q": q})
+            search_urls.append(("DuckDuckGo", "https://duckduckgo.com/html/", {"q": q}))
+
+    for engine, url, params in search_urls:
+        print(f"[WEB_SEARCH:{engine}] {params.get('q')}", flush=True)
+        r = generic_get(url, params=params)
         if not r:
             continue
-        html = r.text
+        page = html.unescape(r.text)
-        for m in re.finditer(r'href="([^"]+)"', html):
+        candidates = set()
-            href = m.group(1)
+        # Bing 结果常在 href="https://..." 或 h="..." 内
+        for m in re.finditer(r'https?://[^"\'<>\s]+', page):
+            href = m.group(0)
             href = href.replace("&amp;", "&")
             if "uddg=" in href:
                 qs = urllib.parse.parse_qs(urllib.parse.urlparse(href).query)
                 href = qs.get("uddg", [href])[0]
-            if not re.search(r"\.(js|py|ts|sh)(\?|$)", href, flags=re.I):
+            # 清理 Bing 跳转尾巴
-                continue
+            href = href.split('&quot;')[0].split('"')[0].split("'")[0]
-            if not any(host in href for host in ["githubusercontent.com", "gitee.com", "gitcode", "agit.ai", "raw"]):
+            candidates.add(href)
-                continue
+        for href in candidates:
+            # 代码文件直链
+            if re.search(r"\.(js|py|ts|sh)(\?|$)", href, flags=re.I) and any(host in href for host in [
+                "githubusercontent.com", "github.com", "gitee.com", "gitcode", "agit.ai", "raw"
+            ]):
+                links[href] = {"url": href}
+            # 仓库页面也先记录，后续可定向扩展
+            elif any(host in href for host in ["gitee.com", "gitcode.com", "agit.ai"]):
+                if re.match(r"https?://[^/]+/[^/?#]+/[^/?#]+/?$", href):
                     links[href] = {"url": href}
     return list(links.values())
 
-
 def get_public_link_content(url: str) -> Tuple[Optional[str], str, str, str]:
     raw_url = url
     html_url = url
@@ -365,6 +389,15 @@ def get_file_uploaded_at(repo_full: str, path: str, fallback: str) -> str:
     return fallback or ""
 
 
+def is_jd_related(content: str, path: str = "", project: str = "") -> bool:
+    hay = f"{path}\n{project}\n{content[:20000]}"
+    lower = hay.lower()
+    for kw in JD_EXCLUDE_KEYWORDS:
+        if kw.lower() in lower:
+            return True
+    return False
+
+
 def looks_like_ql_script(content: str, path: str) -> bool:
     lower = content.lower()
     score = 0
@@ -588,11 +621,14 @@ def add_content_record(
 ) -> bool:
     if not content or not looks_like_ql_script(content, source_path):
         return False
+    project = extract_project(content, source_path)
+    if is_jd_related(content, source_path, project):
+        print(f"  [SKIP_JD] {source_repo}/{source_path}")
+        return False
     sha = normalized_hash(content)
     if sha in seen:
         return False
     category, platform_type, auth_type, evidence = classify(content, source_path)
-    project = extract_project(content, source_path)
     project_dir = OUT_DIR / category / project
     fname = unique_filename(project_dir, uploaded_at, Path(source_path).name, sha)
     saved_path = str((project_dir / fname).relative_to(ROOT))
@@ -711,6 +747,13 @@ def rebuild_index_from_files():
             continue
         category = f"{parts[0]}/{parts[1]}" if parts[0] in {"APP版", "web版", "微信小程序"} else parts[0]
         project = parts[2] if len(parts) >= 4 else "unknown"
+        if is_jd_related(text, rel, project):
+            try:
+                path.unlink()
+                print(f"[REMOVE_JD] {rel}")
+            except Exception:
+                pass
+            continue
         rec = Record(
             category=category,
             project=project,
@@ -739,17 +782,42 @@ def main():
     seen = load_seen_hashes()
     records: List[Record] = []
     files_done = 0
+    github_limited = False
+
+    # 先跑 GitHub；若限流，马上切换其它来源，不阻塞。
     try:
         files_done = collect_github(records, seen, files_done)
     except RuntimeError as e:
         if str(e) == "github_rate_limited":
-            print("[WARN] GitHub 限流，跳过 GitHub 剩余采集，继续其它来源")
+            github_limited = True
+            print("[WARN] GitHub 限流，立即切到 Gitee/公开网页等其它来源", flush=True)
         else:
             raise
+
+    # 其它来源不中断继续采集。
     if files_done < MAX_FILES_TOTAL:
         files_done = collect_gitee(records, seen, files_done)
     if files_done < MAX_FILES_TOTAL:
         files_done = collect_public_links(records, seen, files_done)
+
+    # 其它来源处理完后，如果 GitHub reset 时间已到或等待时间较短，则回头再试一次。
+    if github_limited and files_done < MAX_FILES_TOTAL:
+        now = int(time.time())
+        wait = max(0, GITHUB_RATE_LIMIT_RESET - now + 2)
+        max_wait = int(os.getenv("GITHUB_RETRY_MAX_WAIT", "180"))
+        if wait <= max_wait:
+            print(f"[GITHUB_RETRY] 等待 {wait}s 后回头继续 GitHub", flush=True)
+            time.sleep(wait)
+            try:
+                files_done = collect_github(records, seen, files_done)
+            except RuntimeError as e:
+                if str(e) == "github_rate_limited":
+                    print("[WARN] GitHub 仍限流，本轮结束；下轮继续", flush=True)
+                else:
+                    raise
+        else:
+            print(f"[GITHUB_RETRY] reset 等待时间 {wait}s 超过阈值 {max_wait}s，本轮先不等", flush=True)
+
     if records:
         save_indexes(records)
         update_readme()