diff --git a/.gitea/workflows/update.yml b/.gitea/workflows/update.yml index 8358ff1..ae759e8 100644 --- a/.gitea/workflows/update.yml +++ b/.gitea/workflows/update.yml @@ -34,6 +34,8 @@ jobs: - name: Generate feeds run: python merge.py + env: + RSSHUB_ACCESS_KEY: ${{ secrets.RSSHUB_ACCESS_KEY }} - name: Commit changes run: | diff --git a/README.md b/README.md index 5506c5d..d0c9347 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,10 @@ ## 使用 1. 编辑 [`feeds.yml`](feeds.yml),替换示例订阅源并按需增加分组。 -2. 将仓库推送到 Gitea。 -3. 在仓库设置中启用 Actions,并确保实例上已有可用的 `act_runner`。 -4. 在 Actions 页面手动运行一次 `Update RSS`,确认 `public/*.xml` 被提交。 +2. 在 Gitea 仓库的 Actions Secrets 中创建 `RSSHUB_ACCESS_KEY`,值为 RSSHub 访问密钥。 +3. 将仓库推送到 Gitea。 +4. 在仓库设置中启用 Actions,并确保实例上已有可用的 `act_runner`。 +5. 在 Actions 页面手动运行一次 `Update RSS`,确认 `public/*.xml` 被提交。 订阅地址格式: @@ -38,6 +39,8 @@ groups: - `settings.user_agent`:默认使用 Chrome User-Agent,也可以自行替换。 - `output`:可为分组指定仓库内的相对输出路径。 +凡是主机名为 `rsshub.yuzhen.pub` 的订阅源,请在 `feeds.yml` 中填写不带访问密钥的原始 URL。脚本会读取 `RSSHUB_ACCESS_KEY`,自动改用 HTTPS 并添加或替换 URL 中的 `key` 参数;密钥不会写入生成的 XML 或普通日志。其他域名不会附加该参数。 + 同一个 URL 被多个分组引用时只会请求一次。输出按发布时间倒序排列并截取 `max_items` 条,不进行过滤或去重。内容不变时不会重写文件,也就不会制造空提交。 ## 本地运行 @@ -45,8 +48,10 @@ groups: ```bash python -m venv .venv python -m pip install -r requirements.txt -python merge.py +RSSHUB_ACCESS_KEY=your-key python merge.py python -m unittest discover -s tests -v ``` +如果配置中没有 `rsshub.yuzhen.pub`,本地运行时无需设置这个环境变量。 + 首次成功运行后会创建 `public/` 目录。请将生成的 XML 一并提交,后续源临时不可用时阅读器仍可访问最近一次结果。 diff --git a/merge.py b/merge.py index 81a3114..3737f2b 100644 --- a/merge.py +++ b/merge.py @@ -7,6 +7,7 @@ import argparse import calendar import hashlib import logging +import os import re import sys import tempfile @@ -15,6 +16,7 @@ from datetime import datetime, timezone from email.utils import format_datetime from pathlib import Path from typing import Any, Iterable +from urllib.parse import parse_qsl, quote, quote_plus, urlencode, urlsplit, urlunsplit import xml.etree.ElementTree as ET import feedparser @@ -27,6 +29,8 @@ from urllib3.util.retry import Retry LOG = logging.getLogger("rss-merge") ATOM_NS = "http://www.w3.org/2005/Atom" CONTENT_NS = "http://purl.org/rss/1.0/modules/content/" +RSSHUB_HOST = "rsshub.yuzhen.pub" +RSSHUB_SECRET_ENV = "RSSHUB_ACCESS_KEY" ET.register_namespace("atom", ATOM_NS) ET.register_namespace("content", CONTENT_NS) @@ -113,13 +117,42 @@ def make_session(user_agent: str, retries: int) -> requests.Session: return session +def is_protected_rsshub_url(url: str) -> bool: + try: + return (urlsplit(url).hostname or "").casefold() == RSSHUB_HOST + except ValueError: + return False + + +def add_rsshub_access_key(url: str, access_key: str) -> str: + """Add or replace the key query parameter without changing the configured URL.""" + if not is_protected_rsshub_url(url): + return url + parsed = urlsplit(url) + query = [(name, value) for name, value in parse_qsl(parsed.query, keep_blank_values=True) if name != "key"] + query.append(("key", access_key)) + # Never transmit the access key over plaintext HTTP. + return urlunsplit(("https", parsed.netloc, parsed.path, urlencode(query), parsed.fragment)) + + +def redact_secret(message: str, secret: str) -> str: + if not secret: + return message + for value in {secret, quote(secret, safe=""), quote_plus(secret)}: + if value: + message = message.replace(value, "***") + return message + + def fetch_feed( session: requests.Session, source: Source, timeout: float, max_bytes: int, + rsshub_access_key: str, ) -> list[FeedItem]: - with session.get(source.url, timeout=timeout, stream=True) as response: + request_url = add_rsshub_access_key(source.url, rsshub_access_key) + with session.get(request_url, timeout=timeout, stream=True) as response: response.raise_for_status() chunks: list[bytes] = [] size = 0 @@ -297,6 +330,7 @@ def run(config_path: Path) -> int: ) groups: dict[str, dict[str, Any]] = config["groups"] + rsshub_access_key = os.environ.get(RSSHUB_SECRET_ENV, "") repo_root = config_path.resolve().parent all_sources: dict[str, Source] = {} group_urls: dict[str, list[str]] = {} @@ -313,16 +347,22 @@ def run(config_path: Path) -> int: for source in sources: all_sources.setdefault(source.url, source) + if any(is_protected_rsshub_url(url) for url in all_sources) and not rsshub_access_key: + raise ConfigError(f"配置包含 {RSSHUB_HOST},但环境变量 {RSSHUB_SECRET_ENV} 未设置") + session = make_session(user_agent, retries) + # urllib3 的重试日志可能包含完整查询参数;避免将 access key 写入 Actions 日志。 + logging.getLogger("urllib3").setLevel(logging.CRITICAL) fetched: dict[str, list[FeedItem]] = {} errors: dict[str, str] = {} for url, source in all_sources.items(): try: - fetched[url] = fetch_feed(session, source, timeout, max_bytes) + fetched[url] = fetch_feed(session, source, timeout, max_bytes, rsshub_access_key) LOG.info("已获取 %s(%d 条)", url, len(fetched[url])) except Exception as exc: # One source should produce a useful aggregated error. - errors[url] = str(exc) - LOG.error("获取失败 %s: %s", url, exc) + safe_error = redact_secret(str(exc), rsshub_access_key) + errors[url] = safe_error + LOG.error("获取失败 %s: %s", url, safe_error) if errors and not allow_partial: LOG.error("有 %d 个源失败;为避免临时丢失条目,本次不改写输出。可设置 settings.allow_partial: true 改变此行为。", len(errors)) diff --git a/tests/test_merge.py b/tests/test_merge.py index 3afaeac..713baa7 100644 --- a/tests/test_merge.py +++ b/tests/test_merge.py @@ -3,10 +3,11 @@ from pathlib import Path import sys import unittest import xml.etree.ElementTree as ET +from urllib.parse import parse_qs, urlsplit sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from merge import FeedItem, build_rss, sort_and_limit +from merge import FeedItem, add_rsshub_access_key, build_rss, redact_secret, sort_and_limit def item(title, link, hour, description=""): @@ -22,6 +23,27 @@ def item(title, link, hour, description=""): class MergeTests(unittest.TestCase): + def test_adds_rsshub_key_and_preserves_existing_query(self): + result = add_rsshub_access_key( + "https://rsshub.yuzhen.pub/github/trending/daily?language=python&key=old", + "new key&value", + ) + query = parse_qs(urlsplit(result).query) + self.assertEqual(query, {"language": ["python"], "key": ["new key&value"]}) + + upgraded = add_rsshub_access_key("http://rsshub.yuzhen.pub/feed", "secret") + self.assertEqual(urlsplit(upgraded).scheme, "https") + + def test_does_not_add_key_to_other_hosts(self): + url = "https://example.com/feed.xml?category=tech" + self.assertEqual(add_rsshub_access_key(url, "secret"), url) + + def test_redacts_raw_and_encoded_key(self): + message = "raw=a b&encoded=a+b&percent=a%20b" + self.assertNotIn("a b", redact_secret(message, "a b")) + self.assertNotIn("a+b", redact_secret(message, "a b")) + self.assertNotIn("a%20b", redact_secret(message, "a b")) + def test_sorts_keeps_duplicates_and_limits(self): values = [ item("Older", "https://example.com/old", 1),