From ce95879c2f717b44ef169f8139b76ba89e460663 Mon Sep 17 00:00:00 2001 From: kovawx Date: Wed, 9 Sep 2026 20:27:26 +0800 Subject: [PATCH] feat: automate plugin submissions and upstream release proposals --- .github/ISSUE_TEMPLATE/submit-plugin.yml | 12 +- .github/labels.json | 10 ++ .github/workflows/marketplace.yml | 41 +++++ README.md | 3 +- README.zh-CN.md | 3 +- docs/automation.md | 47 ++++++ docs/automation.zh-CN.md | 47 ++++++ docs/publishing.md | 3 + docs/publishing.zh-CN.md | 3 + scripts/github_api.py | 85 ++++++++++ scripts/marketplace_sync.py | 201 +++++++++++++++++++++++ scripts/release_metadata.py | 79 +++++++++ tests/test_sync.py | 176 ++++++++++++++++++++ 13 files changed, 703 insertions(+), 7 deletions(-) create mode 100644 .github/labels.json create mode 100644 .github/workflows/marketplace.yml create mode 100644 docs/automation.md create mode 100644 docs/automation.zh-CN.md create mode 100644 scripts/github_api.py create mode 100644 scripts/marketplace_sync.py create mode 100644 scripts/release_metadata.py create mode 100644 tests/test_sync.py diff --git a/.github/ISSUE_TEMPLATE/submit-plugin.yml b/.github/ISSUE_TEMPLATE/submit-plugin.yml index 8bf49b4..76c464a 100644 --- a/.github/ISSUE_TEMPLATE/submit-plugin.yml +++ b/.github/ISSUE_TEMPLATE/submit-plugin.yml @@ -1,12 +1,13 @@ name: Submit a plugin / 提交插件 description: Request a listing or release update in the marketplace. / 申请收录插件或更新发行版本。 title: '[Plugin] ' +labels: ['plugin:submission'] body: - type: markdown attributes: value: | - Publish and sign the plugin in its own repository first. Attach marketplace-entry.json or submit a PR using templates/plugin-entry.json. - 先在独立仓库签名发布插件，再附上 marketplace-entry.json 或参照 templates/plugin-entry.json 提交 PR。不要提交源码、安装包或私钥。 + Publish and sign the plugin in its own repository first. Provide the GitHub Release URL for marketplace-entry.json. Automation validates it and opens a review PR. + 先在独立仓库签名发布插件，再提供 marketplace-entry.json 的 GitHub Release 链接，自动化会校验并生成审核 PR。不要提交源码、安装包或私钥。 - type: dropdown id: host attributes: @@ -34,11 +35,12 @@ body: label: Release URL and source commit / 发行地址与源码提交 validations: required: true - - type: textarea + - type: input id: metadata attributes: - label: Release metadata / 发行元数据 - description: Attach marketplace-entry.json or include platform, immutable URL, SHA-256 and byte size for every artifact. / 附上元数据文件，或填写各平台产物的固定地址、SHA-256 与字节数。 + label: Release metadata URL / 发行元数据链接 + description: An immutable GitHub Release asset URL, not a branch file. / 提供 GitHub Release 的固定产物链接，不要使用分支文件地址。 + placeholder: https://github.com/owner/plugin/releases/download/v0.0.1/marketplace-entry.json validations: required: true - type: textarea diff --git a/.github/labels.json b/.github/labels.json new file mode 100644 index 0000000..d90cfbe --- /dev/null +++ b/.github/labels.json @@ -0,0 +1,10 @@ +[ + {"name":"plugin:submission","color":"5319E7","description":"Plugin listing or release submission / 插件收录申请"}, + {"name":"plugin:update","color":"1D76DB","description":"Upstream release update / 上游发行更新"}, + {"name":"host:zboard","color":"0052CC","description":"ZBoard plugin / 面板插件"}, + {"name":"host:znet-sink","color":"0E8A16","description":"ZNet Sink plugin / 客户端插件"}, + {"name":"status:needs-info","color":"D93F0B","description":"Metadata needs correction / 需要补充或修正元数据"}, + {"name":"status:in-review","color":"FBCA04","description":"Validated metadata awaiting maintainer review / 等待维护者审核"}, + {"name":"status:accepted","color":"0E8A16","description":"Release recorded in the main catalog / 已收录到主分支目录"}, + {"name":"status:closed","color":"BFD4F2","description":"Submission or proposal closed without inclusion / 已关闭但未收录"} +] diff --git a/.github/workflows/marketplace.yml b/.github/workflows/marketplace.yml new file mode 100644 index 0000000..7bcf0ff --- /dev/null +++ b/.github/workflows/marketplace.yml @@ -0,0 +1,41 @@ +name: Marketplace synchronization +on: + issues: + types: [opened, edited, reopened, labeled, closed] + pull_request_target: + types: [closed] + branches: [main] + push: + branches: [main] + schedule: + - cron: '23 */6 * * *' + workflow_dispatch: +permissions: {} +concurrency: + group: marketplace-synchronization + cancel-in-progress: false +jobs: + synchronize: + if: github.repository == 'zerodenet/plugins' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write + issues: write + pull-requests: write + steps: + # Privileged events always run reviewed default-branch code, never a + # contributor checkout or anything extracted from a plugin package. + - uses: actions/checkout@v4 + with: + ref: main + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python3 -m unittest discover -s tests + - run: python3 scripts/validate.py + - name: Maintain labels and propose catalog updates + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/marketplace_sync.py diff --git a/README.md b/README.md index b0beb2c..7450897 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A public registry of independently maintained plugins for [ZBoard](https://githu | Host | Registry | Available projects | | --- | --- | --- | -| ZBoard | [catalogs/zboard.json](catalogs/zboard.json) | [OAuth for ZBoard](https://github.com/higanbana986/zboard-oauth) — source available; signed release pending | +| ZBoard | [catalogs/zboard.json](catalogs/zboard.json) | [OAuth for ZBoard](https://github.com/higanbana986/zboard-oauth) — [v0.0.1 released](https://github.com/higanbana986/zboard-oauth/releases/tag/v0.0.1); [listing under review](https://github.com/zerodenet/plugins/issues/1) | | ZNet Sink | [catalogs/znet-sink.json](catalogs/znet-sink.json) | No submissions yet | OAuth connects GitHub, Google and custom OAuth2 / OpenID Connect providers to ZBoard. Its source, tests, configuration guides and release workflows are maintained in its independent repository. ZBoard retains control of account creation, registration policy and sessions. @@ -37,6 +37,7 @@ All maintained registry data and policies live on `main`. Plugin source and bina ## Documentation +- [Submission and update automation](docs/automation.md) - [Registry format](docs/registry-format.md) - [Publishing and review](docs/publishing.md) - [Host responsibilities](docs/governance.md) diff --git a/README.zh-CN.md b/README.zh-CN.md index ff1f26c..c36d2ea 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -8,7 +8,7 @@ | 宿主 | 注册表 | 已有项目 | | --- | --- | --- | -| ZBoard | [catalogs/zboard.json](catalogs/zboard.json) | [OAuth for ZBoard](https://github.com/higanbana986/zboard-oauth) — 源码可用，签名发行待发布 | +| ZBoard | [catalogs/zboard.json](catalogs/zboard.json) | [OAuth for ZBoard](https://github.com/higanbana986/zboard-oauth) — [v0.0.1 已发布](https://github.com/higanbana986/zboard-oauth/releases/tag/v0.0.1)，[收录审核中](https://github.com/zerodenet/plugins/issues/1) | | ZNet Sink | [catalogs/znet-sink.json](catalogs/znet-sink.json) | 暂无提交 | OAuth 将 GitHub、Google 和自定义 OAuth2 / OpenID Connect 提供方接入 ZBoard。源码、测试、配置指南和发行流程均在独立仓库维护。账户创建、注册策略及会话仍由 ZBoard 核心掌管。 @@ -37,6 +37,7 @@ docs/ 注册表格式、发行规范及宿主边界 ## 文档 +- [收录与更新自动化](docs/automation.zh-CN.md) - [注册表格式](docs/registry-format.zh-CN.md) - [发布与审核](docs/publishing.zh-CN.md) - [宿主职责](docs/governance.zh-CN.md) diff --git a/docs/automation.md b/docs/automation.md new file mode 100644 index 0000000..9ea6524 --- /dev/null +++ b/docs/automation.md @@ -0,0 +1,47 @@ +# Marketplace automation + +**English** · [简体中文](automation.zh-CN.md) + +The market proposes catalog changes from published GitHub releases. It does not rebuild plugins, execute their packages, sign releases on behalf of publishers, or merge its own proposals. + +## Submissions + +Publish `marketplace-entry.json` alongside the signed packages, then include its immutable GitHub Release asset URL in the submission form. Existing issues containing that URL are also supported. The workflow checks the release and opens a catalog PR. Correcting the issue reruns validation; one bot comment is maintained instead of adding repeated status messages. + +The metadata must describe exactly one release. Its repository, version and source commit must match the publishing repository and tag, including annotated tags. Every artifact URL, SHA-256 and size must match an asset in that release. Downloads are restricted to GitHub HTTPS release hosting; metadata is limited to 256 KiB. No contributor code is checked out or executed by the privileged workflow. + +These checks establish consistency with GitHub's release records. Maintainers still verify publisher/key ownership, package signatures, requested capabilities, migration behavior and actual host test evidence before merging. A submitted public key does not grant trust. Merge catalog PRs by squash or rebase to keep `main` linear. + +## Labels + +Definitions live in [.github/labels.json](../.github/labels.json). The workflow creates missing labels and updates their colors and descriptions, preserving unrelated labels. + +| Label | Meaning | +| --- | --- | +| `plugin:submission` | Initial listing or submitted release | +| `plugin:update` | Release update found by the scheduled check | +| `host:zboard`, `host:znet-sink` | Host inferred from validated metadata | +| `status:needs-info` | Missing or invalid submission metadata | +| `status:in-review` | Catalog PR awaits review | +| `status:accepted` | Proposal merged or release already present on `main` | +| `status:closed` | Submission or proposal closed without inclusion | + +Labels describe workflow state. Applying `status:accepted` manually cannot write to a catalog or approve an installation. Edits are read again before preparing a PR. A merge closes a linked submission only if its recorded body still matches, or a subsequent reconciliation verifies that its release is already recorded. + +## Upstream updates + +Every six hours, the workflow checks the latest stable GitHub Release for each plugin with an already recorded release and publisher key. Source-only listings are excluded. New releases create separate PRs and append to the existing history. Plugin ID, host, repository and publisher-key changes are rejected. A changed artifact for an existing version is rejected as well. + +A deterministic branch identifies each plugin/version proposal. Repeated runs reuse the existing PR, including closed proposals; rejected updates are not silently reopened. Reopen the proposal manually after resolving review feedback. When several upstream versions appear between scans, the latest stable release is proposed; intermediate versions can be submitted individually. + +The workflow also runs after a main-branch push and can be started from **Actions → Marketplace synchronization → Run workflow**. Commits are made as `github-actions[bot]`. The OAuth repository needs no cross-repository token: publishing its release metadata is sufficient for subsequent scheduled discovery after first inclusion. + +## Repository setup + +Enable **Settings → Actions → General → Workflow permissions → Allow GitHub Actions to create and approve pull requests**. The workflow requests only repository contents, issues and pull-request write permissions; it never approves PRs. Organization policy may control this setting. A permission failure fails the Action rather than labeling a publisher's valid submission as incorrect. + +GitHub may require a maintainer to approve checks on PRs created with `GITHUB_TOKEN`; check the PR's workflow banner. See [GitHub's workflow trigger rules](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow). No personal access token is required by this implementation. + +## Installation feeds + +Merging a PR updates `catalogs/zboard.json` or `catalogs/znet-sink.json`. These remain source catalogs. Publishing and renewing a signed host installation feed is a separate distribution operation with a separate market signing key; this workflow does not turn source JSON into an installable feed. diff --git a/docs/automation.zh-CN.md b/docs/automation.zh-CN.md new file mode 100644 index 0000000..a207248 --- /dev/null +++ b/docs/automation.zh-CN.md @@ -0,0 +1,47 @@ +# 市场自动化 + +[English](automation.md) · **简体中文** + +市场根据已公开的 GitHub Release 提议更新目录，不重新编译插件、不执行安装包、不代替发布者签名，也不自动合并自己的 PR。 + +## 收录申请 + +将 `marketplace-entry.json` 与签名安装包一同发布，在申请表中填写固定的 GitHub Release 产物链接。已有申请正文中的同类链接也可识别。工作流校验发行信息后生成目录 PR；编辑申请会重新校验，机器人维护同一条状态评论，避免重复刷屏。 + +元数据只描述一个版本，仓库、版本、源码提交必须与实际发行及标签一致，支持解析附注标签。每个包的 URL、SHA-256 和大小必须与该次 Release 中的产物记录一致。元数据下载限定 GitHub HTTPS 发行托管域名，最大 256 KiB。带写权限的工作流不检出或执行贡献者代码。 + +这些检查证明元数据与 GitHub 发行记录一致。合并前，维护者仍须核验发布者及公钥归属、安装包签名、所需能力、迁移行为及实际宿主测试证据。提交公钥不授予信任。目录 PR 使用 squash 或 rebase 合并，保持 `main` 线性历史。 + +## 标签 + +定义见 [.github/labels.json](../.github/labels.json)。工作流自动创建缺失标签并同步颜色和说明，保留无关标签。 + +| 标签 | 含义 | +| --- | --- | +| `plugin:submission` | 首次收录或提交的发行版本 | +| `plugin:update` | 定时检查发现的发行更新 | +| `host:zboard`、`host:znet-sink` | 根据有效元数据识别的宿主 | +| `status:needs-info` | 申请元数据缺失或有误 | +| `status:in-review` | 已生成目录 PR，等待审核 | +| `status:accepted` | PR 已合并，或版本已存在于主分支目录 | +| `status:closed` | 申请或提议已关闭，未收录 | + +标签描述流程状态，手工添加 `status:accepted` 不会写入目录，也不能批准宿主安装。准备 PR 前会重新读取申请正文。合并后仅在关联申请正文仍与记录一致时关闭申请，或由后续对账核实其版本已被收录后关闭。 + +## 上游更新 + +每 6 小时检查已有发行记录和发布者公钥的插件仓库，读取最新稳定版 GitHub Release。只有源码的条目不会自动收录。发现新版本后单独生成 PR，并追加到原有发行历史；插件 ID、宿主、仓库、公钥变化以及既有版本的产物变化会被拒绝。 + +每个插件及版本使用固定提议分支，多次运行复用同一个 PR。被关闭的提议不会静默重开，解决审核反馈后由维护者手动重开。如果两次扫描间发布了多个版本，自动提议最新稳定版；中间版本可逐个提交申请。 + +工作流也在 `main` 更新后运行，并支持从 **Actions → Marketplace synchronization → Run workflow** 手动启动。自动提交使用 `github-actions[bot]` 身份。OAuth 仓库无需配置跨仓库 Token；首次收录后，只需随 Release 发布元数据即可被定时发现。 + +## 仓库设置 + +启用 **Settings → Actions → General → Workflow permissions → Allow GitHub Actions to create and approve pull requests**。工作流仅申请仓库内容、Issue 和 PR 写权限，不会批准 PR。组织策略可能控制此设置。权限不足会使 Action 失败，不会把有效申请误标为资料错误。 + +GitHub 可能要求维护者批准由 `GITHUB_TOKEN` 创建的 PR 检查，请查看 PR 上的工作流提示，参见 [GitHub 触发规则](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow)。本实现不需要个人访问 Token。 + +## 安装目录 + +PR 合并后更新 `catalogs/zboard.json` 或 `catalogs/znet-sink.json`，它们仍是源目录。宿主签名安装目录的发布及续签需要独立的市场签名密钥和分发流程；本工作流不会将源 JSON 直接变成可安装目录。 diff --git a/docs/publishing.md b/docs/publishing.md index 454952c..261d499 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -16,6 +16,9 @@ Submit the release using the issue form or PR template. Update only the relevant CI validates repository metadata and never executes submitted packages. Maintainers independently verify publisher/key ownership, release-to-source correspondence, package signatures, digests and host test evidence. A green source-validation check alone is insufficient for an installable listing. + +An issue containing the release metadata URL is validated automatically and linked to a generated review PR. After first inclusion, scheduled checks propose new stable releases without a cross-repository publisher token. Labels track validation, review and inclusion; they do not authorize a merge. See [marketplace automation](automation.md) for setup and failure handling. + ## Signed installation feeds The source catalogs are not directly installable. A distribution publisher must select host- and platform-compatible reviewed artifacts, create the host's signed catalog, serve it over an accepted HTTPS download path and renew it before expiry. Catalog signing uses a market key distinct from plugin publisher keys. diff --git a/docs/publishing.zh-CN.md b/docs/publishing.zh-CN.md index 87c8bfe..22a0e09 100644 --- a/docs/publishing.zh-CN.md +++ b/docs/publishing.zh-CN.md @@ -16,6 +16,9 @@ CI 校验仓库元数据，不执行提交的安装包。维护者独立核实发布者及公钥归属、发行与源码关系、包签名、摘要和宿主测试证据。仅通过源目录校验不足以证明版本可安装。 + +提供发行元数据链接的申请会自动校验并关联生成的审核 PR。首次收录后，定时检查会提议新的稳定版，无需发布者配置跨仓库 Token。标签跟踪校验、审核和收录状态，不授予合并权限。设置及故障处理见[市场自动化](automation.zh-CN.md)。 + ## 签名安装目录 源目录不能直接用于安装。分发发布者需选择已审核、兼容宿主和平台的产物，生成宿主格式的签名目录，使用宿主接受的 HTTPS 下载路径提供文件，并在过期前续签。市场目录密钥与插件发布者密钥分别管理。 diff --git a/scripts/github_api.py b/scripts/github_api.py new file mode 100644 index 0000000..d926167 --- /dev/null +++ b/scripts/github_api.py @@ -0,0 +1,85 @@ +"""Small GitHub client. Tokens are sent only to api.github.com.""" +import json +import os +from urllib.error import HTTPError +from urllib.parse import quote, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from validate import no_duplicates, require + + +class GitHubError(RuntimeError): + def __init__(self, status, path): + self.status = status + super().__init__(f'GitHub HTTP {status}: {path}') + + +class AssetRedirect(HTTPRedirectHandler): + max_redirections = 4 + max_repeats = 2 + + def redirect_request(self, req, fp, code, msg, headers, newurl): + check_asset_url(newurl) + return Request(newurl, headers={'User-Agent': 'zerodenet-marketplace'}) + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *args): + return None + + +def check_asset_url(url): + parsed = urlsplit(url) + require(parsed.scheme == 'https' and parsed.hostname in { + 'github.com', 'release-assets.githubusercontent.com', 'objects.githubusercontent.com' + } and not parsed.username and not parsed.password and not parsed.fragment + and parsed.port in (None, 443), 'asset URL must use GitHub HTTPS release hosting') + + +def decode_json(raw): + return json.loads(raw, object_pairs_hook=no_duplicates) + + +class GitHub: + def __init__(self, token=None): + self.token = token if token is not None else os.environ.get('GH_TOKEN', '') + + def api(self, path, method='GET', data=None): + require(path.startswith('/') and not path.startswith('//'), 'invalid GitHub API path') + headers = {'Accept': 'application/vnd.github+json', 'User-Agent': 'zerodenet-marketplace', + 'X-GitHub-Api-Version': '2022-11-28'} + if self.token: + headers['Authorization'] = 'Bearer ' + self.token + raw = None if data is None else json.dumps(data).encode() + if raw is not None: + headers['Content-Type'] = 'application/json' + req = Request('https://api.github.com' + path, raw, headers, method=method) + try: + with build_opener(NoRedirect).open(req, timeout=30) as response: + body = response.read(8 * 1024 * 1024 + 1) + require(len(body) <= 8 * 1024 * 1024, 'GitHub response too large') + return decode_json(body) if body else None + except HTTPError as error: + raise GitHubError(error.code, path) from None + + def pages(self, path): + separator = '&' if '?' in path else '?' + for page in range(1, 11): + rows = self.api(f'{path}{separator}per_page=100&page={page}') + yield from rows + if len(rows) < 100: + return + raise ValueError('pagination limit exceeded; narrow the repository query') + + def asset(self, url, size): + check_asset_url(url) + require(0 < size <= 256 * 1024, 'release metadata must be at most 256 KiB') + request = Request(url, headers={'User-Agent': 'zerodenet-marketplace'}) + with build_opener(AssetRedirect).open(request, timeout=30) as response: + data = response.read(256 * 1024 + 1) + require(len(data) == size, 'metadata download size mismatch') + return data + + +def segment(value): + return quote(str(value), safe='') diff --git a/scripts/marketplace_sync.py b/scripts/marketplace_sync.py new file mode 100644 index 0000000..147c995 --- /dev/null +++ b/scripts/marketplace_sync.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Maintain submission labels and propose reviewed catalog updates.""" +import base64 +import hashlib +import json +import os +from pathlib import Path +import re + +from github_api import GitHub, GitHubError, decode_json, segment +from release_metadata import append_entry, release_entry, submission +from validate import HOSTS, require + +ROOT = Path(__file__).resolve().parents[1] +STATES = {'status:needs-info', 'status:in-review', 'status:accepted', 'status:closed'} + + +class Marketplace: + def __init__(self, github, repository): + require(re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repository), 'invalid market repository') + self.github, self.repository = github, repository + self.prefix = f'/repos/{repository}' + + def labels(self): + existing = {label['name']: label for label in self.github.pages(self.prefix + '/labels')} + for label in json.loads((ROOT / '.github/labels.json').read_text()): + old = existing.get(label['name']) + if old is None: + self.github.api(self.prefix + '/labels', 'POST', label) + elif any(old.get(field) != label[field] for field in ('color', 'description')): + self.github.api(self.prefix + '/labels/' + segment(label['name']), 'PATCH', label) + + def status(self, number, state, extra=()): + path = f'{self.prefix}/issues/{number}' + current = self.github.api(path) + names = {label['name'] for label in current['labels']} + desired_hosts = set(extra) & {'host:zboard', 'host:znet-sink'} + stale_hosts = (names & {'host:zboard', 'host:znet-sink'}) - desired_hosts if desired_hosts else set() + for label in sorted(((names & STATES) - {state}) | stale_hosts): + self.github.api(path + '/labels/' + segment(label), 'DELETE') + add = ({state} | set(extra)) - names + if add: + self.github.api(path + '/labels', 'POST', {'labels': sorted(add)}) + + def comment(self, number, text): + marker = '' + path = f'{self.prefix}/issues/{number}/comments' + body = marker + '\n' + text + for comment in self.github.pages(path): + if comment['user']['login'] == 'github-actions[bot]' and comment['body'].startswith(marker): + if comment['body'] != body: + self.github.api(f'{self.prefix}/issues/comments/{comment["id"]}', 'PATCH', {'body': body}) + return + self.github.api(path, 'POST', {'body': body}) + + def catalog(self, host, ref='main'): + require(host in HOSTS, 'unsupported host') + record = self.github.api(f'{self.prefix}/contents/catalogs/{host}.json?ref={segment(ref)}') + require(record.get('encoding') == 'base64', 'catalog is not a GitHub text blob') + return decode_json(base64.b64decode(record['content'])), record['sha'] + + def propose(self, host, entry, issue=None): + base = self.github.api(self.prefix + '/git/ref/heads/main')['object']['sha'] + catalog, blob = self.catalog(host, base) + updated = append_entry(catalog, entry, host) + if catalog == updated: + return None + version = entry['releases'][0]['version'] + branch = f'automation/{host}/{entry["id"]}/{version}' + prs = self.github.api(self.prefix + '/pulls?state=all&head=' + segment(self.repository.split('/')[0] + ':' + branch)) + if prs: + # Never reopen a rejected proposal or overwrite a reviewed branch. + return prs[0] + encoded = base64.b64encode((json.dumps(updated, indent=2, ensure_ascii=False) + '\n').encode()).decode() + try: + self.github.api(self.prefix + '/git/refs', 'POST', {'ref': 'refs/heads/' + branch, 'sha': base}) + except GitHubError as error: + if error.status != 422: + raise + # A previous run may have created its branch before failing to open + # a PR. Reuse only an exact proposed catalog or an untouched base. + previous, _ = self.catalog(host, branch) + require(previous == catalog or previous == updated, 'existing automation branch changed; inspect it before retrying') + previous, previous_blob = self.catalog(host, branch) + if previous != updated: + require(previous_blob == blob, 'automation branch no longer matches main') + self.github.api(f'{self.prefix}/contents/catalogs/{host}.json', 'PUT', { + 'message': f'registry: add {entry["id"]} {version}', 'branch': branch, + 'sha': blob, 'content': encoded, + 'author': {'name': 'github-actions[bot]', 'email': '41898282+github-actions[bot]@users.noreply.github.com'}, + }) + link = entry['repository'] + '/releases/tag/' + version + body = (f'Update `{entry["id"]}` for `{host}` from [upstream {version}]({link}).\n\n' + 'Verified: metadata SHA-256, tag/source commit, release asset sizes and GitHub-recorded digests, ' + 'host schema, publisher continuity and immutable release history. Plugin binaries were not executed or rebuilt. ' + 'Package signatures and publisher ownership still require maintainer review.\n\n' + '已校验发行元数据、源码提交、GitHub 产物摘要与大小及历史不可变性；安装包验签、公钥归属和实际宿主测试仍由维护者审核。\n\n' + '- [ ] Publisher/key ownership and package signatures reviewed / 发布者归属及包签名已核验\n' + '- [ ] Capabilities, migrations and host test evidence reviewed / 能力、迁移及宿主测试证据已审核\n') + if issue: + digest = hashlib.sha256((issue.get('body') or '').encode()).hexdigest() + body += f'\nSubmission: #{issue["number"]}\n\n' + pr = self.github.api(self.prefix + '/pulls', 'POST', { + 'title': f'registry: {entry["id"]} {version}', 'head': branch, 'base': 'main', 'body': body, + }) + self.status(pr['number'], 'status:in-review', ['host:' + host, 'plugin:submission' if issue else 'plugin:update']) + return pr + + def intake(self, issue): + if issue.get('pull_request') or issue['state'] != 'open': + return + number = issue['number'] + try: + repository, tag = submission(issue.get('body')) + host, entry = release_entry(self.github, repository, tag) + # Recheck after network work; an edit or close must invalidate this run. + fresh = self.github.api(f'{self.prefix}/issues/{number}') + if fresh.get('body') != issue.get('body') or fresh['state'] != 'open': + return + pr = self.propose(host, entry, issue) + except GitHubError as error: + if error.status != 404: + raise + self.status(number, 'status:needs-info', ['plugin:submission']) + self.comment(number, 'The public release or tag is unavailable. / 请确认发行已公开且标签存在。') + return + except (ValueError, KeyError, TypeError) as error: + self.status(number, 'status:needs-info', ['plugin:submission']) + self.comment(number, 'Metadata needs attention / 请修正发行元数据：\n\n' + str(error)) + return + if pr is None: + self.status(number, 'status:accepted', ['plugin:submission', 'host:' + host]) + self.comment(number, 'This release is recorded in the main catalog. / 此版本已收录到主分支目录。') + self.github.api(f'{self.prefix}/issues/{number}', 'PATCH', {'state': 'closed', 'state_reason': 'completed'}) + else: + state = 'status:in-review' if pr['state'] == 'open' else 'status:closed' + self.status(number, state, ['plugin:submission', 'host:' + host]) + self.comment(number, f'Marketplace proposal / 市场更新 PR：{pr["html_url"]}\n\n' + 'Maintainer review is required before inclusion. / 合并审核通过后才会收录。') + + def reconcile(self): + for issue in self.github.pages(self.prefix + '/issues?state=open'): + if not issue.get('pull_request') and (issue['title'].startswith('[Plugin]') or any(label['name'] == 'plugin:submission' for label in issue['labels'])): + self.intake(issue) + + def updates(self): + failures = [] + for host in sorted(HOSTS): + catalog, _ = self.catalog(host) + for old in catalog['plugins']: + if not old['releases'] or old['publisher']['public_key'] is None: + continue + repository = old['repository'].removeprefix('https://github.com/') + try: + latest = self.github.api(f'/repos/{repository}/releases/latest') + actual_host, entry = release_entry(self.github, repository, latest['tag_name']) + require(actual_host == host and entry['id'] == old['id'], 'upstream host or plugin ID changed') + self.propose(host, entry) + except (ValueError, KeyError, TypeError, GitHubError) as error: + failures.append(f'{host}/{old["id"]}: {error}') + if failures: + raise RuntimeError('Some upstream releases require attention:\n' + '\n'.join(failures)) + + def closed_pr(self, pr): + if pr['head']['repo'] is None or pr['head']['repo']['full_name'] != self.repository or not pr['head']['ref'].startswith('automation/'): + return + self.status(pr['number'], 'status:accepted' if pr.get('merged') else 'status:closed') + match = re.search(r'', pr.get('body') or '') + if match: + issue = self.github.api(f'{self.prefix}/issues/{match[1]}') + if hashlib.sha256((issue.get('body') or '').encode()).hexdigest() == match[2]: + if pr.get('merged'): + self.intake(issue) + else: + self.status(issue['number'], 'status:closed') + + +def main(): + github = GitHub() + require(github.token, 'GH_TOKEN is required') + market = Marketplace(github, os.environ['GITHUB_REPOSITORY']) + event = decode_json(Path(os.environ['GITHUB_EVENT_PATH']).read_bytes()) + market.labels() + event_name = os.environ['GITHUB_EVENT_NAME'] + if event_name == 'issues': + issue = github.api(f'{market.prefix}/issues/{event["issue"]["number"]}') + if issue['title'].startswith('[Plugin]') or any(label['name'] == 'plugin:submission' for label in issue['labels']): + if issue['state'] == 'closed': + if not any(label['name'] == 'status:accepted' for label in issue['labels']): + market.status(issue['number'], 'status:closed') + else: + market.intake(issue) + elif event_name == 'pull_request_target': + market.closed_pr(event['pull_request']) + else: + market.reconcile() + market.updates() + + +if __name__ == '__main__': + main() diff --git a/scripts/release_metadata.py b/scripts/release_metadata.py new file mode 100644 index 0000000..046ca99 --- /dev/null +++ b/scripts/release_metadata.py @@ -0,0 +1,79 @@ +"""Validate GitHub release facts and append metadata without running plugins.""" +import copy +import hashlib +import re + +from github_api import decode_json, segment +from validate import HOSTS, TAG, require, validate, validate_entry, validate_transition + +METADATA_URL = re.compile(r'https://github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)/releases/download/(' + TAG + r')/marketplace-entry\.json') + + +def submission(body): + urls = {match.group(0) for match in METADATA_URL.finditer(body or '')} + require(len(urls) == 1, 'Provide exactly one GitHub Release marketplace-entry.json URL / 请提供唯一的发行元数据链接') + match = METADATA_URL.fullmatch(urls.pop()) + return match[1], match[2] + + +def infer_host(entry): + hosts = {host for release in entry.get('releases', []) for host in HOSTS if host in release.get('requires', {})} + require(len(hosts) == 1, 'metadata must identify exactly one host') + return hosts.pop() + + +def release_entry(github, repository, tag): + release = github.api(f'/repos/{repository}/releases/tags/{segment(tag)}') + require(not release['draft'] and not release['prerelease'] and release['tag_name'] == tag, 'only published stable releases are synchronized') + assets = list(github.pages(f'/repos/{repository}/releases/{release["id"]}/assets')) + records = [asset for asset in assets if asset['name'] == 'marketplace-entry.json'] + require(len(records) == 1, 'release needs one marketplace-entry.json asset') + record = records[0] + expected_url = f'https://github.com/{repository}/releases/download/{tag}/marketplace-entry.json' + require(record['browser_download_url'] == expected_url, 'metadata asset repository or tag mismatch') + raw = github.asset(expected_url, record['size']) + require(record.get('digest') == 'sha256:' + hashlib.sha256(raw).hexdigest(), 'metadata digest does not match GitHub release asset') + entry = decode_json(raw) + host = infer_host(entry) + validate_entry(entry, host) + require(entry['repository'] == f'https://github.com/{repository}', 'metadata must belong to the publishing repository') + require(len(entry['releases']) == 1, 'release metadata must describe exactly its own release') + version = entry['releases'][0] + require(version['version'] == tag == entry['source']['version'], 'release version differs from metadata') + require(version['source_commit'] == entry['source']['commit'], 'source commits differ') + ref = github.api(f'/repos/{repository}/git/ref/tags/{segment(tag)}')['object'] + for _ in range(4): + if ref['type'] != 'tag': + break + ref = github.api(f'/repos/{repository}/git/tags/{ref["sha"]}')['object'] + require(ref['type'] == 'commit' and ref['sha'] == version['source_commit'], 'tag does not resolve to the declared source commit') + by_url = {asset['browser_download_url']: asset for asset in assets} + for artifact in version['artifacts']: + asset = by_url.get(artifact['url']) + require(asset is not None and asset['size'] == artifact['size'] + and asset.get('digest') == 'sha256:' + artifact['sha256'], 'package must match a GitHub asset size and digest in this release') + return host, entry + + +def append_entry(catalog, entry, host): + validate(catalog, host) + validate_entry(entry, host) + result = copy.deepcopy(catalog) + old = next((item for item in result['plugins'] if item['id'] == entry['id']), None) + candidate = copy.deepcopy(entry) + if old: + require(old['repository'] == entry['repository'], 'repository reassignment requires a separate review') + releases = {release['version']: release for release in old['releases']} + for release in entry['releases']: + if release['version'] in releases: + require(releases[release['version']] == release, 'published release metadata cannot change') + if all(release['version'] in releases for release in entry['releases']): + require(old['publisher'] == entry['publisher'], 'publisher key cannot change') + return result + candidate['releases'] = old['releases'] + [release for release in entry['releases'] if release['version'] not in releases] + result['plugins'][result['plugins'].index(old)] = candidate + else: + result['plugins'].append(candidate) + validate(result, host) + validate_transition(catalog, result) + return result diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..b331be7 --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,176 @@ +import base64 +import copy +import hashlib +import json +from pathlib import Path +import sys +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'scripts')) +from github_api import check_asset_url +from marketplace_sync import Marketplace +from release_metadata import append_entry, release_entry, submission + + +class ReleaseAPI: + def __init__(self): + self.entry = json.loads((ROOT / 'templates/plugin-entry.json').read_text()) + self.repo = self.entry['repository'].removeprefix('https://github.com/') + self.tag = self.entry['source']['version'] + self.base = f'https://github.com/{self.repo}/releases/download/{self.tag}/' + self.entry['releases'] = self.entry['releases'][:1] + self.entry['releases'][0]['source_commit'] = self.entry['source']['commit'] + for a in self.entry['releases'][0]['artifacts']: + a['url'] = self.base + a['platform'] + '.zbplugin' + self.assets = [{'name': 'package.zbplugin', 'browser_download_url': a['url'], 'size': a['size'], 'digest': 'sha256:' + a['sha256']} for a in self.entry['releases'][0]['artifacts']] + self.commit = self.entry['source']['commit'] + self.draft = False + self.refresh() + + def refresh(self): + self.raw = json.dumps(self.entry).encode() + self.metadata = {'name': 'marketplace-entry.json', 'browser_download_url': self.base + 'marketplace-entry.json', 'size': len(self.raw), 'digest': 'sha256:' + hashlib.sha256(self.raw).hexdigest()} + + def api(self, path): + if '/releases/tags/' in path: + return {'id': 1, 'tag_name': self.tag, 'draft': self.draft, 'prerelease': False} + if '/git/ref/tags/' in path: + return {'object': {'type': 'commit', 'sha': self.commit}} + raise AssertionError(path) + + def pages(self, path): + return iter([self.metadata] + self.assets) + + def asset(self, url, size): + assert url == self.metadata['browser_download_url'] + return self.raw + + +class SyncTest(unittest.TestCase): + def test_proposals_only_write_the_host_catalog_and_reuse_existing_prs(self): + class API: + def __init__(self): + self.catalog = {'schema_version': 1, 'host': 'zboard', 'plugins': []} + self.proposals = [] + self.writes = [] + self.branch_catalog = None + def api(self, path, method='GET', data=None): + if method != 'GET': self.writes.append((path, method, data)) + if path.endswith('/git/ref/heads/main'): return {'object': {'sha': 'a' * 40}} + if '/contents/catalogs/zboard.json?ref=' in path: + value = self.catalog if path.endswith('a' * 40) else self.branch_catalog + return {'encoding': 'base64', 'sha': 'b' * 40, 'content': base64.b64encode(json.dumps(value).encode()).decode()} + if '/pulls?state=all&head=' in path: return self.proposals + if path.endswith('/git/refs'): + self.branch_catalog = copy.deepcopy(self.catalog); return {} + if path.endswith('/contents/catalogs/zboard.json') and method == 'PUT': + self.branch_catalog = json.loads(base64.b64decode(data['content'])); return {} + if path.endswith('/pulls') and method == 'POST': + result = {'number': 2, 'state': 'open', 'html_url': 'https://github.com/zerodenet/plugins/pull/2'} + self.proposals.append(result); return result + if path.endswith('/issues/2'): return {'labels': []} + if path.endswith('/issues/2/labels'): return {} + raise AssertionError((path, method, data)) + api = API() + market = Marketplace(api, 'zerodenet/plugins') + entry = ReleaseAPI().entry + first = market.propose('zboard', entry) + writes = len(api.writes) + self.assertEqual(market.propose('zboard', entry), first) + self.assertEqual(len(api.writes), writes) + puts = [path for path, method, _ in api.writes if method == 'PUT'] + self.assertEqual(puts, ['/repos/zerodenet/plugins/contents/catalogs/zboard.json']) + api.proposals[0]['state'] = 'closed' + self.assertEqual(market.propose('zboard', entry)['state'], 'closed') + self.assertEqual(len(api.writes), writes) + + def test_existing_submission_and_form_link_are_supported(self): + url = 'https://github.com/higanbana986/zboard-oauth/releases/download/v0.0.1/marketplace-entry.json' + self.assertEqual(submission(f'### Release metadata URL\n\n{url}'), ('higanbana986/zboard-oauth', 'v0.0.1')) + self.assertEqual(submission(f'- [marketplace-entry.json]({url})'), ('higanbana986/zboard-oauth', 'v0.0.1')) + for body in ('https://example.com/metadata.json', url + '\n' + url.replace('v0.0.1', 'v0.0.2'), ''): + with self.assertRaises(ValueError): + submission(body) + + def test_release_verifies_source_and_all_server_recorded_artifacts(self): + api = ReleaseAPI() + host, entry = release_entry(api, api.repo, api.tag) + self.assertEqual(host, 'zboard') + self.assertEqual(entry, api.entry) + for change in ( + lambda a: setattr(a, 'draft', True), + lambda a: setattr(a, 'commit', 'f' * 40), + lambda a: a.metadata.update(digest='sha256:' + 'f' * 64), + lambda a: a.assets[0].update(size=2), + lambda a: a.assets[0].update(digest='sha256:' + 'f' * 64), + lambda a: a.assets[0].update(browser_download_url='https://example.com/file.zbplugin'), + ): + with self.subTest(change=change): + api = ReleaseAPI(); change(api) + with self.assertRaises(ValueError): + release_entry(api, api.repo, api.tag) + + def test_metadata_cannot_claim_another_repository(self): + api = ReleaseAPI() + api.entry['repository'] = 'https://github.com/other/plugin'; api.refresh() + with self.assertRaises(ValueError): + release_entry(api, api.repo, api.tag) + + def test_merge_is_idempotent_and_preserves_immutable_history(self): + api = ReleaseAPI() + empty = {'schema_version': 1, 'host': 'zboard', 'plugins': []} + first = append_entry(empty, api.entry, 'zboard') + self.assertEqual(append_entry(first, api.entry, 'zboard'), first) + next_entry = copy.deepcopy(api.entry) + next_entry['source']['version'] = next_entry['releases'][0]['version'] = 'v0.0.2' + second = append_entry(first, next_entry, 'zboard') + self.assertEqual(second['plugins'][0]['releases'][0], api.entry['releases'][0]) + self.assertEqual(len(second['plugins'][0]['releases']), 2) + self.assertEqual(first['plugins'][0]['releases'], api.entry['releases']) + for mutate in ( + lambda e: e['publisher'].update(public_key=base64.b64encode(b'x' * 32).decode()), + lambda e: e['publisher'].update(id='other'), + lambda e: e.update(repository='https://github.com/other/plugin'), + lambda e: e['releases'][0]['artifacts'][0].update(size=2), + ): + changed = copy.deepcopy(api.entry); mutate(changed) + with self.assertRaises(ValueError): + append_entry(first, changed, 'zboard') + + def test_source_only_listing_gets_its_first_release(self): + api = ReleaseAPI() + old = copy.deepcopy(api.entry); old['releases'] = []; old['publisher']['public_key'] = None + current = {'schema_version': 1, 'host': 'zboard', 'plugins': [old]} + result = append_entry(current, api.entry, 'zboard') + self.assertEqual(len(result['plugins']), 1) + self.assertEqual(len(result['plugins'][0]['releases']), 1) + + def test_downloads_never_redirect_to_arbitrary_hosts_or_credentials(self): + for url in ('http://github.com/a', 'https://evil.example/a', 'https://127.0.0.1/a', 'https://user:secret@github.com/a', 'https://github.com:8443/a', 'https://github.com.evil.example/a'): + with self.assertRaises(ValueError): + check_asset_url(url) + check_asset_url('https://release-assets.githubusercontent.com/a?temporary=token') + + def test_status_only_replaces_owned_status_labels(self): + class API: + def __init__(self): self.calls = [] + def api(self, path, method='GET', data=None): + self.calls.append((path, method, data)) + return {'labels': [{'name': n} for n in ['priority:high', 'status:needs-info', 'host:zboard']]} + api = API() + Marketplace(api, 'zerodenet/plugins').status(1, 'status:in-review', ['host:zboard']) + self.assertEqual([call[1] for call in api.calls], ['GET', 'DELETE', 'POST']) + self.assertTrue(api.calls[1][0].endswith('status%3Aneeds-info')) + self.assertEqual(api.calls[2][2], {'labels': ['status:in-review']}) + + def test_privileged_workflow_never_checks_out_contributor_code(self): + workflow = (ROOT / '.github/workflows/marketplace.yml').read_text() + self.assertIn('ref: main', workflow) + self.assertIn('persist-credentials: false', workflow) + self.assertNotIn('github.event.issue.body', workflow) + self.assertNotIn('github.event.pull_request.head.sha', workflow) + + +if __name__ == '__main__': + unittest.main()