Merge branch 'master' into agent/onboarding-modal-flow

This commit is contained in:
Tianyi Cui
2026-08-13 17:03:59 +08:00
committed by GitHub
9 changed files with 88 additions and 57 deletions

View File

@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write .agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md
2026-07-22-evidence-based-larger-hosted-runners.md: 84c951809891b4936549a2f429dc7efc99833c1b
2026-07-22-evidence-based-larger-hosted-runners.zh.md: bb5b585da26ffd15b5a1096cd40293f64fb69c79
2026-07-22-evidence-based-larger-hosted-runners.md: e0d919851d99eac6539a25c63c9baeb49f76335f
2026-07-22-evidence-based-larger-hosted-runners.zh.md: 673bd7643506f022b640d14918dd7c883fb60e36

View File

@@ -52,7 +52,7 @@ The process-bound coverage project contains exactly five suite files. Thirty-two
Complete serial Linux, macOS, and Windows references run only when `master` moves. Pull requests use the enterprise required path plus standard-hosted compatibility jobs, while other larger-runner sizes run only by manual dispatch.
An additional serial Linux reference runs on the in-house self-hosted pool (`vm-backup` label: a 64-core VM with six always-on systemd-managed runner instances) on every `master` push. It is a hot-standby drill, not a required check: each run re-proves that the persistent VM can execute the complete unsharded aggregate. The actual switch is pre-wired: the three required Linux jobs resolve their pool through the writer-manageable `DSH_CI_FAILOVER` repository variable, so an outage response is setting one variable and re-running — no merge, which would be deadlocked behind the failing checks themselves ([runbook](2026-07-26-ci-failover-runbook.md)). The standby lane is push-triggered, so it always executes the base branch's workflow definition. Under failover, however, `pull_request` jobs do reach these runners with the PR merge ref's own workflow definition — the trust boundary is repository membership (the repository is private with forking disabled, and the selectors exclude Dependabot), as the [failover runbook](2026-07-26-ci-failover-runbook.md) records.
An additional serial Linux reference runs on the in-house self-hosted pool (`vm-backup` label: a 64-core VM with six always-on systemd-managed runner instances) on every `master` push. It is a hot-standby drill, not a required check: each run re-proves that the persistent VM can execute the complete unsharded aggregate. The actual switch is pre-wired: the three required Linux jobs resolve their pool through the writer-manageable `DSH_CI_FAILOVER_LINUX` repository variable, so an outage response is setting one variable and re-running — no merge, which would be deadlocked behind the failing checks themselves ([runbook](2026-07-26-ci-failover-runbook.md)). The standby lane is push-triggered, so it always executes the base branch's workflow definition. Under failover, however, `pull_request` jobs do reach these runners with the PR merge ref's own workflow definition — the trust boundary is repository membership (the repository is private with forking disabled, and the selectors exclude Dependabot), as the [failover runbook](2026-07-26-ci-failover-runbook.md) records.
## Alternatives considered

View File

@@ -52,7 +52,7 @@ Windows 仓库工作在超过 16 核后收益很小,但 32 核池可以让完
只有在 `master` 移动时,才运行完整的 Linux、macOS 和 Windows 串行参考。拉取请求使用企业级运行器必需路径和标准托管兼容性作业,其他大型运行器规格仅通过手动触发运行。
另有一条串行 Linux 参考在每次 `master` 推送时运行于公司自有的自托管池(`vm-backup` 标签:一台 64 核虚拟机,运行 6 个常驻的 systemd 管理运行器实例)。它是热备演练而非必需检查:每次运行都重新证明这台持久化虚拟机能够执行完整的未分片聚合流程。实际切换机制已预先布线:三个必需 Linux 作业通过写入权限持有者可管理的仓库变量 `DSH_CI_FAILOVER` 解析运行器池,因此故障响应就是设置一个变量并重跑——无需合并(合并本身会被正在失败的检查阻塞,形成死锁)([切换手册](2026-07-26-ci-failover-runbook.md))。该热备通道由 push 触发,执行的始终是基础分支自身的工作流定义。但需要注意:故障切换期间,`pull_request` 作业确实会带着 PR merge 引用自带的工作流定义到达这些运行器——信任边界是仓库成员资格(仓库为私有且禁用 fork选择器排除 Dependabot详见[故障切换手册](2026-07-26-ci-failover-runbook.md)的记录。
另有一条串行 Linux 参考在每次 `master` 推送时运行于公司自有的自托管池(`vm-backup` 标签:一台 64 核虚拟机,运行 6 个常驻的 systemd 管理运行器实例)。它是热备演练而非必需检查:每次运行都重新证明这台持久化虚拟机能够执行完整的未分片聚合流程。实际切换机制已预先布线:三个必需 Linux 作业通过写入权限持有者可管理的仓库变量 `DSH_CI_FAILOVER_LINUX` 解析运行器池,因此故障响应就是设置一个变量并重跑——无需合并(合并本身会被正在失败的检查阻塞,形成死锁)([切换手册](2026-07-26-ci-failover-runbook.md))。该热备通道由 push 触发,执行的始终是基础分支自身的工作流定义。但需要注意:故障切换期间,`pull_request` 作业确实会带着 PR merge 引用自带的工作流定义到达这些运行器——信任边界是仓库成员资格(仓库为私有且禁用 fork选择器排除 Dependabot详见[故障切换手册](2026-07-26-ci-failover-runbook.md)的记录。
## 曾考虑的替代方案

View File

@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write .agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md
2026-07-26-ci-failover-runbook.md: 90ef2905bec86911697e552c0c1eac96ea0d3a18
2026-07-26-ci-failover-runbook.zh.md: 5c8fd0e90ae31e3622ae6b49217591a54b84c351
2026-07-26-ci-failover-runbook.md: b4522e623ffb76f3fd33d242b21c2d1d9ff2eadf
2026-07-26-ci-failover-runbook.zh.md: 58ba7ffee013d38f06afe097362f5c23f04b8121

View File

@@ -6,11 +6,11 @@ English | [中文](2026-07-26-ci-failover-runbook.zh.md)
## Problem
The three required Linux worker jobs in [CI](../../../../.github/workflows/ci.yml) (`node 24 / static`, `node 24 / coverage`, `node 24 / snapshots and artifacts`) run on the hosted enterprise 32-core pools; the required verdict job that aggregates them (`all checks passed`) runs on standard `ubuntu-latest`; the independent native Windows job (`windows node 24 / native complete`) runs on the hosted `dsh-windows-2025-16core` larger runner. When the enterprise pools degrade — jobs queue indefinitely or the enterprise labels vanish — every open pull request becomes unmergeable, and the ordinary recovery of merging a fix is itself deadlocked behind the very required checks that cannot run. **Scope: this switch recovers an enterprise Linux-pool outage AND a hosted Windows-pool outage.** The verdict's other required dependencies (`node-compat`, `python-sdk`, `windows`) stay on standard hosted runners by design (the portable boundary); in a broader GitHub-hosted capacity failure that also takes out the standard pools, those dependencies still block `all checks passed`. An outage therefore needs a switch any responder with repository write access can throw without merging anything.
The three required Linux worker jobs in [CI](../../../../.github/workflows/ci.yml) (`node 24 / static`, `node 24 / coverage`, `node 24 / snapshots and artifacts`) run on the hosted enterprise 32-core pools; the required verdict job that aggregates them (`all checks passed`) runs on standard `ubuntu-latest`; the independent native Windows job (`windows node 24 / native complete`) runs on the hosted `dsh-windows-2025-16core` larger runner. When the enterprise pools degrade — jobs queue indefinitely or the enterprise labels vanish — every open pull request becomes unmergeable, and the ordinary recovery of merging a fix is itself deadlocked behind the very required checks that cannot run. **Scope: two independent switches, one per platform.** `DSH_CI_FAILOVER_LINUX` recovers an enterprise Linux-pool outage (the three required Linux workers plus the `all checks passed` verdict); `DSH_CI_FAILOVER_WINDOWS` recovers a hosted Windows-pool outage (the native Windows job). A Linux-pool outage need not retarget the native Windows job and vice versa. The verdict's other required dependencies (`node-compat`, `python-sdk`, `windows`) stay on standard hosted runners by design (the portable boundary); in a broader GitHub-hosted capacity failure that also takes out the standard pools, those dependencies still block `all checks passed`. An outage therefore needs a switch any responder with repository write access can throw without merging anything.
## Decision
Each of the three required Linux worker jobs, the independent native Windows job, and the `all checks passed` verdict job — which would otherwise stay queued on the failed pool even after every worker passed — resolves its runner pool through the `DSH_CI_FAILOVER` repository variable. Unset (normal), they run on the hosted enterprise pools. Set to `selfhosted` by any repository writer, all five retarget onto the in-house self-hosted pools: the Linux jobs and verdict onto the `vm-backup` pool, coverage and snapshot concurrency drop to shared-VM bounds, and the hosted-path pnpm cache restores are skipped; the native Windows job onto the `dsh-win-ci` pool. The switch is writer-manageable repository state, not a merge, so it works while every check is red. The in-house pools' readiness is continuously re-proven by the `serial / linux (self-hosted standby)` and `serial / windows (self-hosted standby)` lanes, which run the complete unsharded aggregates on every master push.
Each of the three required Linux worker jobs, the independent native Windows job, and the `all checks passed` verdict job — which would otherwise stay queued on the failed pool even after every worker passed — resolves its runner pool through a repository variable, and the switch is split by platform so an outage on one platform does not retarget the other. The three Linux workers and the `all checks passed` verdict (whose `needs` are the required Linux workers and which runs on the `vm-backup` pool) resolve through `DSH_CI_FAILOVER_LINUX`; the native Windows job resolves through `DSH_CI_FAILOVER_WINDOWS`. Unset (normal), they run on the hosted enterprise pools. Set to `selfhosted` by any repository writer, the corresponding jobs retarget onto the in-house self-hosted pool: under `DSH_CI_FAILOVER_LINUX`, the Linux jobs and verdict move onto the `vm-backup` pool, coverage and snapshot concurrency drop to shared-VM bounds, and the hosted-path pnpm cache restores are skipped; under `DSH_CI_FAILOVER_WINDOWS`, the native Windows job moves onto the `dsh-win-ci` pool. Each switch is writer-manageable repository state, not a merge, so it works while every check is red. The in-house pools' readiness is continuously re-proven by the `serial / linux (self-hosted standby)` and `serial / windows (self-hosted standby)` lanes, which run the complete unsharded aggregates on every master push.
`ci.yml` exempts exactly one event from `cancel-in-progress` (`${{ github.event_name != 'push' }}`), so one master push does not cancel the drill still running from the previous one. Each drill runs its complete unsharded aggregate with one gate worker, which takes longer than the interval between master merges; under unconditional cancellation a drill is superseded before reaching a verdict and the lane yields no readiness evidence for a responder to check.
@@ -28,13 +28,15 @@ The decision belongs at workflow level because cancellation applies to the whole
### Switch (any repository writer, ~1 minute, no merge)
1. Repository **Settings → Secrets and variables → Actions → Variables → New repository variable**: name `DSH_CI_FAILOVER`, value `selfhosted`.
The two switches are independent: flip only the one whose platform is degraded.
1. Repository **Settings → Secrets and variables → Actions → Variables → New repository variable**: name `DSH_CI_FAILOVER_LINUX` (Linux pool outage) or `DSH_CI_FAILOVER_WINDOWS` (Windows pool outage), value `selfhosted`.
2. Retrigger the required jobs so they re-resolve their pool. Jobs already **queued** for the hosted labels do not retarget and cannot be re-run in place, so for the documented indefinite-queue outage, cancel the stuck run and re-run all jobs, or push a new commit; "Re-run failed jobs" only helps once a job has actually failed rather than queued.
3. That is the entire switch. Under failover the workflow also, automatically: drops `DSH_COVERAGE_MAX_WORKERS` to 8 and `DSH_SNAPSHOT_MAX_CONCURRENCY` to 12 (sized for six always-on instances: worst case 6 × 8 = 48 coverage workers on the 64-core VM) (shared-VM contention bounds), and skips the hosted-path pnpm cache restores (the VM's persistent store serves warm installs).
3. That is the entire switch. Under Linux failover the workflow also, automatically: drops `DSH_COVERAGE_MAX_WORKERS` to 8 and `DSH_SNAPSHOT_MAX_CONCURRENCY` to 12 (sized for six always-on instances: worst case 6 × 8 = 48 coverage workers on the 64-core VM) (shared-VM contention bounds), and skips the hosted-path pnpm cache restores (the VM's persistent store serves warm installs). The Windows switch has no such concurrency or cache branches; it only retargets the native Windows job's pool.
#**Dependabot exception.** All five selectors deliberately exclude `dependabot[bot]`: under failover, Dependabot PRs stay queued for the hosted pool rather than executing dependency-supplied code on the persistent VMs. A Dependabot PR that remains queued during an outage is expected behavior, not a failed switch; it completes when the hosted pool recovers.
#**Dependabot exception.** Both switches' selectors deliberately exclude `dependabot[bot]`: under failover, Dependabot PRs stay queued for the hosted pool rather than executing dependency-supplied code on the persistent VMs. A Dependabot PR that remains queued during an outage is expected behavior, not a failed switch; it completes when the hosted pool recovers.
**Who can flip the variable.** GitHub's API lets any collaborator with write access manage repository variables, so the switch is writer-level, not strictly admin-only. In this repository's trust model that is not an escalation: the runner groups admit all workflows of this private, fork-disabled repository (a deliberate trade to make PR-ref failover possible at all), so any writer could already reach the VMs by pushing a branch workflow. The boundary against untrusted code is repository membership; the variable only routes work for members.
**Who can flip the variable.** GitHub's API lets any collaborator with write access manage repository variables, so each switch is writer-level, not strictly admin-only. In this repository's trust model that is not an escalation: the runner groups admit all workflows of this private, fork-disabled repository (a deliberate trade to make PR-ref failover possible at all), so any writer could already reach the VMs by pushing a branch workflow. The boundary against untrusted code is repository membership; the variables only route work for members.
## Capacity during failover
@@ -43,18 +45,18 @@ Six always-on instances absorb normal PR traffic (the pool's steady-state load i
### Switch back
Delete the `DSH_CI_FAILOVER` variable (or set it to anything other than `selfhosted`). New runs resolve back to the hosted enterprise pools. Remove any extra instances that were registered during the incident.
Delete the `DSH_CI_FAILOVER_LINUX` or `DSH_CI_FAILOVER_WINDOWS` variable (or set it to anything other than `selfhosted`). New runs resolve back to the hosted enterprise pools. Remove any extra instances that were registered during the incident.
### Trust boundary
The variable is writer-manageable repository state; a pull request event itself can neither set it nor read a different value into effect, and the selector expressions live in workflow definitions. Note that under failover, `pull_request` runs execute the PR merge ref's own workflow definition — the boundary against untrusted code is repository membership (private, forking disabled, Dependabot excluded by the selectors), not the variable. Note on runner-group policy: pinning the runner group to the master-ref workflow is **incompatible** with this failover — the four failover jobs are `pull_request` runs evaluated from PR merge refs, and a master-pinned group leaves them queued (observed live on 2026-07-27; the group was widened to all workflows of this repository to unblock the switch). A stricter runner-side policy therefore costs PR failover; the shipped posture accepts repository-scoped, all-workflow group access.
The variables are writer-manageable repository state; a pull request event itself can neither set them nor read a different value into effect, and the selector expressions live in workflow definitions. Note that under failover, `pull_request` runs execute the PR merge ref's own workflow definition — the boundary against untrusted code is repository membership (private, forking disabled, Dependabot excluded by the selectors), not the variable. Note on runner-group policy: pinning the runner group to the master-ref workflow is **incompatible** with this failover — the five failover jobs are `pull_request` runs evaluated from PR merge refs, and a master-pinned group leaves them queued (observed live on 2026-07-27; the group was widened to all workflows of this repository to unblock the switch). A stricter runner-side policy therefore costs PR failover; the shipped posture accepts repository-scoped, all-workflow group access.
## Alternatives considered
**Merge a workflow change to switch pools.** Rejected because the outage that motivates the switch is exactly the state in which no PR can merge: the required checks are the ones failing. A repository variable is writer-manageable state that takes effect on re-run without a merge.
**Keep the self-hosted pool always in the required path.** Rejected because it trades hosted-pool availability for the in-house VM's, moving a single point of failure rather than adding a fallback. The variable keeps the hosted pools primary and the self-hosted pool a proven, one-action standby.
**Keep the self-hosted pool always in the required path.** Rejected because it trades hosted-pool availability for the in-house VM's, moving a single point of failure rather than adding a fallback. The variables keep the hosted pools primary and the self-hosted pools proven, one-action standbys; splitting them by platform means an outage on one platform does not retarget the other.
## Consequences
Recovering from a hosted-pool outage is a single variable (any writer) plus a re-run, with no merge on the critical path. The cost is a second runner topology to keep working: the standby lane exercises it on every master push so the failover target never goes stale, and the concurrency and cache-restore branches in `ci.yml` carry a `selfhosted` leg that must stay in step with the hosted leg.
Recovering from a hosted-pool outage is flipping the affected platform's variable (any writer) plus a re-run, with no merge on the critical path. The cost is a second runner topology per platform to keep working: the standby lanes exercise them on every master push so the failover targets never go stale, and the concurrency and cache-restore branches in `ci.yml` carry a `selfhosted` leg (Linux only) that must stay in step with the hosted leg. Splitting the switch by platform adds one more variable to manage but bounds the blast radius of each switch to the jobs of a single platform.

View File

@@ -6,11 +6,11 @@ Status: implemented
## 问题
[CI](../../../../.github/workflows/ci.yml) 中三个必需的 Linux 工作作业(`node 24 / static``node 24 / coverage``node 24 / snapshots and artifacts`)运行在托管的企业级 32 核池上;聚合它们的必需判定作业(`all checks passed`)运行在标准 `ubuntu-latest` 上;独立的原生 Windows 作业(`windows node 24 / native complete`)运行在托管的 `dsh-windows-2025-16core` 大型运行器上。当企业池发生故障——作业无限排队或企业标签消失——所有开启的拉取请求都无法合并,而"合并一个修复"这一常规恢复手段本身正被那些无法运行的必需检查死锁。**适用范围:本切换恢复的是企业级 Linux 池故障与托管 Windows 池故障。**判定作业的其余必需依赖(`node-compat``python-sdk``windows`)按设计留在标准托管运行器上(可移植边界);若更大范围的 GitHub 托管容量故障连标准池一并击倒,这些依赖仍会阻塞 `all checks passed`。因此故障需要一个任何具备仓库写权限的响应者都能在不合并任何代码的情况下触发的开关。
[CI](../../../../.github/workflows/ci.yml) 中三个必需的 Linux 工作作业(`node 24 / static``node 24 / coverage``node 24 / snapshots and artifacts`)运行在托管的企业级 32 核池上;聚合它们的必需判定作业(`all checks passed`)运行在标准 `ubuntu-latest` 上;独立的原生 Windows 作业(`windows node 24 / native complete`)运行在托管的 `dsh-windows-2025-16core` 大型运行器上。当企业池发生故障——作业无限排队或企业标签消失——所有开启的拉取请求都无法合并,而"合并一个修复"这一常规恢复手段本身正被那些无法运行的必需检查死锁。**适用范围:两个独立开关,每个平台一个。**`DSH_CI_FAILOVER_LINUX` 恢复企业级 Linux 池故障(三个必需的 Linux 工作作业加 `all checks passed` 判定作业);`DSH_CI_FAILOVER_WINDOWS` 恢复托管 Windows 池故障(原生 Windows 作业。Linux 池故障无需重定向原生 Windows 作业,反之亦然。判定作业的其余必需依赖(`node-compat``python-sdk``windows`)按设计留在标准托管运行器上(可移植边界);若更大范围的 GitHub 托管容量故障连标准池一并击倒,这些依赖仍会阻塞 `all checks passed`。因此故障需要一个任何具备仓库写权限的响应者都能在不合并任何代码的情况下触发的开关。
## 决策
三个必需的 Linux 工作作业、独立的原生 Windows 作业,以及 `all checks passed` 判定作业(若不随切换,即使全部工作作业通过,它仍会滞留在故障池的队列中)——各自通过仓库变量 `DSH_CI_FAILOVER` 解析运行器池。变量不存在(正常)时它们运行在托管企业池上;由任何具备写权限的协作者设为 `selfhosted` 时,五个作业全部切换到公司自有的自托管池Linux 作业与判定作业切到 `vm-backup` 池,覆盖率与快照的并发降到共享虚拟机上限,并跳过托管路径的 pnpm 缓存恢复;原生 Windows 作业切到 `dsh-win-ci` 池。个开关是写者可管理的仓库状态而非一次合并,因此在所有检查都是红色时仍然有效。自有池的就绪状态由 `serial / linux (self-hosted standby)``serial / windows (self-hosted standby)` 通道持续验证——每次 master 推送都在其上运行完整的未分片聚合流程。
三个必需的 Linux 工作作业、独立的原生 Windows 作业,以及 `all checks passed` 判定作业(若不随切换,即使全部工作作业通过,它仍会滞留在故障池的队列中)——各自通过仓库变量解析运行器池,且开关按平台拆分,使一个平台的故障不会重定向另一个平台。三个 Linux 工作作业与 `all checks passed` 判定作业(其 `needs` 是必需的 Linux 工作作业,且运行在 `vm-backup` 池上)通过 `DSH_CI_FAILOVER_LINUX` 解析;原生 Windows 作业通过 `DSH_CI_FAILOVER_WINDOWS` 解析。变量不存在(正常)时它们运行在托管企业池上;由任何具备写权限的协作者设为 `selfhosted` 时,对应作业切换到公司自有的自托管池:`DSH_CI_FAILOVER_LINUX` 下,Linux 作业与判定作业切到 `vm-backup` 池,覆盖率与快照的并发降到共享虚拟机上限,并跳过托管路径的 pnpm 缓存恢复;`DSH_CI_FAILOVER_WINDOWS` 下,原生 Windows 作业切到 `dsh-win-ci` 池。个开关是写者可管理的仓库状态而非一次合并,因此在所有检查都是红色时仍然有效。自有池的就绪状态由 `serial / linux (self-hosted standby)``serial / windows (self-hosted standby)` 通道持续验证——每次 master 推送都在其上运行完整的未分片聚合流程。
`ci.yml` 只豁免一个事件不做取消(`${{ github.event_name != 'push' }}`),因此一次 master 推送不会取消上一次推送留下的、仍在运行的演练。每次演练以单门禁工作进程执行完整的未分片聚合流程,耗时长于 master 合并的间隔;在无条件取消下,演练会在得出结论前被后续运行取代,该通道无法产出供响应者查看的就绪证据。
@@ -28,13 +28,15 @@ Status: implemented
### 切换步骤(任何具备写权限的协作者,约 1 分钟,无需合并)
1. 仓库 **Settings → Secrets and variables → Actions → Variables → New repository variable**:名称 `DSH_CI_FAILOVER`,值 `selfhosted`
两个开关相互独立:只切换发生故障的那个平台
1. 仓库 **Settings → Secrets and variables → Actions → Variables → New repository variable**:名称 `DSH_CI_FAILOVER_LINUX`Linux 池故障)或 `DSH_CI_FAILOVER_WINDOWS`Windows 池故障),值 `selfhosted`
2. 重新触发必需作业,使其重新解析运行器池。已经为托管标签**排队**的作业不会重定向,也无法原地 re-run因此对于本手册所述的无限排队故障应取消卡住的运行并 re-run all jobs或推送一个新提交“Re-run failed jobs”只有在作业真正失败而非仍在排队时才有用。
3. 切换到此完成。故障切换状态下工作流还会自动:把 `DSH_COVERAGE_MAX_WORKERS` 降为 8、`DSH_SNAPSHOT_MAX_CONCURRENCY` 降为 12按 6 个常驻实例定容最坏情况下6 × 8 = 48 个覆盖率工作进程运行在 64 核虚拟机上)(共享虚拟机的争抢上限),并跳过托管路径的 pnpm 缓存恢复(虚拟机的持久 store 直接提供热安装)。
3. 切换到此完成。Linux 故障切换状态下工作流还会自动:把 `DSH_COVERAGE_MAX_WORKERS` 降为 8、`DSH_SNAPSHOT_MAX_CONCURRENCY` 降为 12按 6 个常驻实例定容最坏情况下6 × 8 = 48 个覆盖率工作进程运行在 64 核虚拟机上)(共享虚拟机的争抢上限),并跳过托管路径的 pnpm 缓存恢复(虚拟机的持久 store 直接提供热安装)。Windows 开关没有这类并发或缓存分支;它只重定向原生 Windows 作业的运行器池。
#**Dependabot 例外。**五个选择器都刻意排除了 `dependabot[bot]`故障切换期间Dependabot 拉取请求继续在托管池排队,而不是把依赖项提供的代码放到持久化虚拟机上执行。故障期间 Dependabot PR 持续排队是预期行为而非切换失败;托管池恢复后它会自行完成。
#**Dependabot 例外。**两个开关的选择器都刻意排除了 `dependabot[bot]`故障切换期间Dependabot 拉取请求继续在托管池排队,而不是把依赖项提供的代码放到持久化虚拟机上执行。故障期间 Dependabot PR 持续排队是预期行为而非切换失败;托管池恢复后它会自行完成。
**谁能扳动这个变量。**GitHub 的 API 允许任何具有写权限的协作者管理仓库变量,因此开关实际是写者级而非严格的管理员级。在本仓库的信任模型下这并不构成升权runner group 接纳本私有、禁 fork 仓库的全部工作流(这是让 PR 引用的故障切换得以成立的刻意取舍),因此任何写者本就可以通过推送分支工作流触达这台虚拟机。抵御不可信代码的边界是仓库成员资格;变量只是为成员路由工作。
**谁能扳动这个变量。**GitHub 的 API 允许任何具有写权限的协作者管理仓库变量,因此每个开关实际是写者级而非严格的管理员级。在本仓库的信任模型下这并不构成升权runner group 接纳本私有、禁 fork 仓库的全部工作流(这是让 PR 引用的故障切换得以成立的刻意取舍),因此任何写者本就可以通过推送分支工作流触达这台虚拟机。抵御不可信代码的边界是仓库成员资格;变量只是为成员路由工作。
## 切换期间的容量
@@ -43,18 +45,18 @@ Status: implemented
### 切回
删除 `DSH_CI_FAILOVER` 变量(或改为 `selfhosted` 以外的任何值),新的运行即解析回托管企业池。若故障期间追加注册过实例,将其移除。
删除 `DSH_CI_FAILOVER_LINUX``DSH_CI_FAILOVER_WINDOWS` 变量(或改为 `selfhosted` 以外的任何值),新的运行即解析回托管企业池。若故障期间追加注册过实例,将其移除。
### 信任边界
变量是写者可管理的仓库状态;`pull_request` 事件本身既不能设置它,也不能让不同的值生效,选择器表达式存在于工作流定义中。需要注意:故障切换期间,`pull_request` 运行执行的是 PR merge 引用自带的工作流定义——抵御不可信代码的边界是仓库成员资格(私有、禁 fork、选择器排除 Dependabot而非该变量。关于 runner group 策略的说明:把 runner group 绑定到 master 引用的工作流与本故障切换机制**不兼容**——个故障切换作业是从 PR merge 引用求值的 `pull_request` 运行master 绑定的组会让它们持续排队2026-07-27 实际故障中亲历;当时将组放宽为本仓库全部工作流才疏通了切换)。更严格的运行器侧策略以牺牲 PR 故障切换为代价;当前采用的形态是仓库范围、全工作流的组访问。
这些变量是写者可管理的仓库状态;`pull_request` 事件本身既不能设置它,也不能让不同的值生效,选择器表达式存在于工作流定义中。需要注意:故障切换期间,`pull_request` 运行执行的是 PR merge 引用自带的工作流定义——抵御不可信代码的边界是仓库成员资格(私有、禁 fork、选择器排除 Dependabot而非该变量。关于 runner group 策略的说明:把 runner group 绑定到 master 引用的工作流与本故障切换机制**不兼容**——个故障切换作业是从 PR merge 引用求值的 `pull_request` 运行master 绑定的组会让它们持续排队2026-07-27 实际故障中亲历;当时将组放宽为本仓库全部工作流才疏通了切换)。更严格的运行器侧策略以牺牲 PR 故障切换为代价;当前采用的形态是仓库范围、全工作流的组访问。
## 曾考虑的替代方案
**通过合并一次工作流改动来切换池。** 否决,因为触发切换的故障状态恰恰是任何 PR 都无法合并的状态:必需检查正是失败的那些。仓库变量是写者可管理的状态,重跑即生效,无需合并。
**让自托管池长期处于必需路径中。** 否决,因为这是拿托管池的可用性去换自有虚拟机的可用性,只是搬移了单点故障而非增加回退。变量让托管池保持主路径,自托管池作为一个经过验证、一步即可启用的热备。
**让自托管池长期处于必需路径中。** 否决,因为这是拿托管池的可用性去换自有虚拟机的可用性,只是搬移了单点故障而非增加回退。这些变量让托管池保持主路径,自托管池作为一个经过验证、一步即可启用的热备;按平台拆分意味着一个平台的故障不会重定向另一个平台
## 后果
从托管池故障中恢复只需一个变量(任何写者可设)加一次重跑,关键路径上没有合并。代价是要维护第二套运行器拓扑:热备通道在每次 master 推送时都运行它,避免故障切换目标变得陈旧;而 `ci.yml` 中的并发与缓存恢复分支带有一条 `selfhosted` 支路,必须与托管支路保持同步。
从托管池故障中恢复只需切换受影响平台的变量(任何写者可设)加一次重跑,关键路径上没有合并。代价是每个平台都要维护第二套运行器拓扑:热备通道在每次 master 推送时都运行它,避免故障切换目标变得陈旧;而 `ci.yml` 中的并发与缓存恢复分支带有一条 `selfhosted` 支路(仅 Linux,必须与托管支路保持同步。按平台拆分开关多了一个需要管理的变量,但把每个开关的影响范围限定在单个平台的作业上。

2
.github/AGENTS.md vendored
View File

@@ -1,3 +1,3 @@
# AGENTS.md — GitHub Actions
Run jobs on Windows runners (`windows-*` labels) under native `pwsh`. The pull-request `windows` job is the deliberate exception: it runs Windows Node under Wine on hosted Linux and blocks `all checks passed`; `windows-native` runs automatically on `windows-2025` (or the self-hosted `[self-hosted, dsh-win-ci, windows]` pool under `DSH_CI_FAILOVER=selfhosted`) but reports independently. The master `serial-windows` standby continuously validates the self-hosted failover target — see the [failover runbook](../.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md).
Run jobs on Windows runners (`windows-*` labels) under native `pwsh`. The pull-request `windows` job is the deliberate exception: it runs Windows Node under Wine on hosted Linux and blocks `all checks passed`; `windows-native` runs automatically on `windows-2025` (or the self-hosted `[self-hosted, dsh-win-ci, windows]` pool under `DSH_CI_FAILOVER_WINDOWS=selfhosted`) but reports independently. The master `serial-windows` standby continuously validates the self-hosted failover target — see the [failover runbook](../.agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md).

View File

@@ -49,8 +49,8 @@ jobs:
# build-backed consumer tail. The consumer job owns the only Linux build so
# all three jobs enter runner allocation independently.
#
# FAILOVER: each Linux enterprise job resolves its pool through the
# DSH_CI_FAILOVER repository variable. Unset (normal), the expressions
# FAILOVER (Linux): each Linux enterprise job resolves its pool through the
# DSH_CI_FAILOVER_LINUX repository variable. Unset (normal), the expressions
# pick the hosted enterprise pools below. Setting the variable to
# 'selfhosted' (repo Settings → Actions → Variables; writer-manageable
# repository state — not PR-editable, no merge required) retargets all
@@ -58,11 +58,12 @@ jobs:
# vm-backup pool and re-running the failed jobs is the entire switch —
# see .agents/notes/implemented/process/2026-07-26-ci-failover-runbook.md. The
# in-house pool's readiness is re-proven on every master push by the
# serial-linux-selfhosted standby lane below.
# serial-linux-selfhosted standby lane below. The Windows failover switch is
# the separate DSH_CI_FAILOVER_WINDOWS variable on the windows-native job below.
node-24:
if: github.event_name == 'pull_request'
runs-on: >-
${{ vars.DSH_CI_FAILOVER == 'selfhosted'
${{ vars.DSH_CI_FAILOVER_LINUX == 'selfhosted'
&& github.event.pull_request.user.login != 'dependabot[bot]'
&& fromJSON('["self-hosted", "linux", "x64", "vm-backup"]')
|| 'dsh-ubuntu-24-04-16core' }}
@@ -96,7 +97,7 @@ jobs:
# compression and upload on the paid latency-critical path. Skipped
# under failover — see the coverage lane's identical rationale.
- uses: actions/cache/restore@v4
if: vars.DSH_CI_FAILOVER != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'
if: vars.DSH_CI_FAILOVER_LINUX != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'
with:
path: ${{ steps.pnpm-store.outputs.path }}
key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
@@ -114,7 +115,7 @@ jobs:
node-24-coverage:
if: github.event_name == 'pull_request'
runs-on: >-
${{ vars.DSH_CI_FAILOVER == 'selfhosted'
${{ vars.DSH_CI_FAILOVER_LINUX == 'selfhosted'
&& github.event.pull_request.user.login != 'dependabot[bot]'
&& fromJSON('["self-hosted", "linux", "x64", "vm-backup"]')
|| 'dsh-ubuntu-24-04-16core' }}
@@ -124,7 +125,7 @@ jobs:
# shares one 64-core VM across six always-on runner instances, so each
# instance may use eight while keeping the worst case at 8 × 6 = 48
# workers; process-bound suites remain isolated in forks.
DSH_COVERAGE_MAX_WORKERS: ${{ vars.DSH_CI_FAILOVER == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]' && '8' || '6' }}
DSH_COVERAGE_MAX_WORKERS: ${{ vars.DSH_CI_FAILOVER_LINUX == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]' && '8' || '6' }}
DSH_GATE_CONCURRENCY: '3'
steps:
- uses: actions/checkout@v6
@@ -151,7 +152,7 @@ jobs:
# already serves warm installs, while restoring the hosted archive
# would spend ~52 s pulling ~180 MB into that populated store.
- uses: actions/cache/restore@v4
if: vars.DSH_CI_FAILOVER != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'
if: vars.DSH_CI_FAILOVER_LINUX != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'
with:
path: ${{ steps.pnpm-store.outputs.path }}
key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
@@ -177,7 +178,7 @@ jobs:
node-24-consumers:
if: github.event_name == 'pull_request'
runs-on: >-
${{ vars.DSH_CI_FAILOVER == 'selfhosted'
${{ vars.DSH_CI_FAILOVER_LINUX == 'selfhosted'
&& github.event.pull_request.user.login != 'dependabot[bot]'
&& fromJSON('["self-hosted", "linux", "x64", "vm-backup"]')
|| 'dsh-ubuntu-24-04-16core' }}
@@ -188,7 +189,7 @@ jobs:
DSH_OXLINT_THREADS: '8'
DSH_PUBLINT_CONCURRENCY: '8'
# Failover halves snapshot concurrency for the shared 64-core VM.
DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ vars.DSH_CI_FAILOVER == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]' && '12' || '32' }}
DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ vars.DSH_CI_FAILOVER_LINUX == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]' && '12' || '32' }}
steps:
- uses: actions/checkout@v6
with:
@@ -212,7 +213,7 @@ jobs:
# Skipped under failover — see the coverage lane's identical rationale.
- uses: actions/cache/restore@v4
if: vars.DSH_CI_FAILOVER != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'
if: vars.DSH_CI_FAILOVER_LINUX != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'
with:
path: ${{ steps.pnpm-store.outputs.path }}
key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
@@ -222,7 +223,7 @@ jobs:
# Pull requests restore the cache normally produced by serial-linux on
# master; they do not pay compression and upload on the required path.
- uses: actions/cache/restore@v4
if: vars.DSH_CI_FAILOVER != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'
if: vars.DSH_CI_FAILOVER_LINUX != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'
with:
path: ~/.cache/ms-playwright
key: ${{ runner.os }}-playwright-${{ hashFiles('pnpm-lock.yaml') }}
@@ -243,13 +244,13 @@ jobs:
exit "$sandbox_status"
- name: Install Playwright Chromium and hosted dependencies
if: vars.DSH_CI_FAILOVER != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'
if: vars.DSH_CI_FAILOVER_LINUX != 'selfhosted' || github.event.pull_request.user.login == 'dependabot[bot]'
run: pnpm --filter @deepseek-ai/dsh-web-frontend exec playwright install --with-deps chromium
# The persistent VM image owns Playwright's Linux system packages; do
# not mutate the shared host with apt on every failover run.
- name: Install Playwright Chromium on the failover VM
if: vars.DSH_CI_FAILOVER == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]'
if: vars.DSH_CI_FAILOVER_LINUX == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]'
run: pnpm --filter @deepseek-ai/dsh-web-frontend exec playwright install chromium
- name: Run compatibility, snapshot, and artifact gates
@@ -437,13 +438,16 @@ jobs:
# its own unmasked conclusion but is deliberately absent from
# all-checks-passed.needs, so it never delays or changes that required
# verdict. Under normal operation it runs on the hosted larger runner; under
# failover (DSH_CI_FAILOVER=selfhosted) it retargets onto the in-house
# self-hosted Windows pool. Dependabot PRs are excluded from the self-hosted
# pool and stay queued for the hosted runner — see the failover runbook.
# Windows failover (DSH_CI_FAILOVER_WINDOWS=selfhosted) it retargets onto the
# in-house self-hosted Windows pool. Dependabot PRs are excluded from the
# self-hosted pool and stay queued for the hosted runner — see the failover
# runbook. This Windows switch is independent of the Linux
# DSH_CI_FAILOVER_LINUX variable that retargets the three required Linux jobs
# and the all-checks-passed verdict above.
windows-native:
if: github.event_name == 'pull_request'
runs-on: >-
${{ vars.DSH_CI_FAILOVER == 'selfhosted'
${{ vars.DSH_CI_FAILOVER_WINDOWS == 'selfhosted'
&& github.event.pull_request.user.login != 'dependabot[bot]'
&& fromJSON('["self-hosted", "dsh-win-ci", "windows"]')
|| 'dsh-windows-2025-16core' }}
@@ -558,7 +562,7 @@ jobs:
# re-runs the complete unsharded aggregate on the persistent 64-core VM,
# continuously proving that environment can take over a required lane if
# the hosted pools degrade (the switch is then setting the writer-manageable
# DSH_CI_FAILOVER variable — see the failover runbook, no merge required).
# DSH_CI_FAILOVER_LINUX variable — see the failover runbook, no merge required).
# Push-triggered, so this lane always executes the base branch's own
# workflow definition. (Under failover, pull_request jobs do reach these
# runners with the PR merge ref's workflow — the boundary there is
@@ -644,11 +648,11 @@ jobs:
# move re-runs the complete unsharded Windows gate inventory on the persistent
# VM, continuously proving that environment can take over the required
# `windows` lane if the hosted pool degrades (the switch is setting the
# writer-manageable DSH_CI_FAILOVER variable — see the failover runbook, no
# merge required). Push-triggered, so this lane always executes the base
# branch's own workflow definition. Non-blocking for pull requests; absent
# from all-checks-passed.needs by design — the required `windows` job owns
# the PR verdict. No cache steps because the VM's persistent pnpm store
# writer-manageable DSH_CI_FAILOVER_WINDOWS variable — see the failover
# runbook, no merge required). Push-triggered, so this lane always executes
# the base branch's own workflow definition. Non-blocking for pull requests;
# absent from all-checks-passed.needs by design — the required `windows` job
# owns the PR verdict. No cache steps because the VM's persistent pnpm store
# and tool caches make them redundant (and saving here would poison the
# hosted cache namespace with self-hosted paths).
serial-windows:
@@ -911,11 +915,13 @@ jobs:
all-checks-passed:
name: all checks passed
# This bookkeeping-only verdict must not depend on custom-pool
# provisioning — and under failover it follows the same selector as the
# worker jobs it aggregates, so a standard-hosted outage cannot strand
# the branch-protection verdict either.
# provisioning — and under Linux failover it follows the same selector as
# the worker jobs it aggregates, so a standard-hosted outage cannot strand
# the branch-protection verdict either. It retargets with the Linux switch
# (DSH_CI_FAILOVER_LINUX), not the Windows one, because it aggregates the
# required Linux workers and runs on the vm-backup pool.
runs-on: >-
${{ vars.DSH_CI_FAILOVER == 'selfhosted'
${{ vars.DSH_CI_FAILOVER_LINUX == 'selfhosted'
&& github.event.pull_request.user.login != 'dependabot[bot]'
&& fromJSON('["self-hosted", "linux", "x64", "vm-backup"]')
|| 'ubuntu-latest' }}

View File

@@ -34,14 +34,20 @@ describe('CI workflow', () => {
|| !isRecord(workflow.jobs['windows-native'])
|| !isRecord(workflow.jobs['wine-apt-cache'])
|| !isRecord(workflow.jobs['serial-windows'])
|| !isRecord(workflow.jobs['node-24'])
|| !isRecord(workflow.jobs['node-24-coverage'])
|| !isRecord(workflow.jobs['node-24-consumers'])
|| !isRecord(workflow.jobs['all-checks-passed'])) {
throw new TypeError('CI workflow must define windows, windows-native, wine-apt-cache, serial-windows, and all-checks-passed jobs')
throw new TypeError('CI workflow must define windows, windows-native, wine-apt-cache, serial-windows, node-24, node-24-coverage, node-24-consumers, and all-checks-passed jobs')
}
const windows = workflow.jobs.windows
const windowsNative = workflow.jobs['windows-native']
const wineAptCache = workflow.jobs['wine-apt-cache']
const serialWindows = workflow.jobs['serial-windows']
const node24 = workflow.jobs['node-24']
const node24Coverage = workflow.jobs['node-24-coverage']
const node24Consumers = workflow.jobs['node-24-consumers']
const aggregate = workflow.jobs['all-checks-passed']
if (!Array.isArray(windows.steps) || !Array.isArray(aggregate.needs)) {
throw new TypeError('Windows job must define steps and the aggregate must define needs')
@@ -57,8 +63,10 @@ describe('CI workflow', () => {
expect(commandSteps.some(step => step.run.includes('wine-windows-gates.sh'))).toBe(true)
// windows-native: non-blocking native job with failover, runs windows-complete.
// Its pool is resolved by the Windows-specific switch.
expect(typeof windowsNative['runs-on']).toBe('string')
expect(windowsNative['runs-on']).toContain('DSH_CI_FAILOVER')
expect(windowsNative['runs-on']).toContain('DSH_CI_FAILOVER_WINDOWS')
expect(windowsNative['runs-on']).not.toContain('DSH_CI_FAILOVER_LINUX')
expect(windowsNative['runs-on']).toContain('self-hosted')
expect(windowsNative['runs-on']).toContain('dsh-win-ci')
expect(windowsNative['runs-on']).toContain('dsh-windows-2025-16core')
@@ -82,6 +90,19 @@ describe('CI workflow', () => {
expect(aggregate.needs).toContain('windows')
expect(aggregate.needs).not.toContain('windows-native')
expect(aggregate.needs).not.toContain('serial-windows')
// Linux failover is a separate switch: the three required Linux workers
// and the verdict job resolve their pool through DSH_CI_FAILOVER_LINUX,
// never the Windows switch.
for (const [jobName, job] of [['node-24', node24], ['node-24-coverage', node24Coverage], ['node-24-consumers', node24Consumers]] as const) {
expect(typeof job['runs-on']).toBe('string')
expect(job['runs-on'], `${jobName} runs-on must use the Linux failover switch`).toContain('DSH_CI_FAILOVER_LINUX')
expect(job['runs-on'], `${jobName} runs-on must not use the Windows failover switch`).not.toContain('DSH_CI_FAILOVER_WINDOWS')
expect(job['runs-on']).toContain('vm-backup')
}
expect(aggregate['runs-on']).toContain('DSH_CI_FAILOVER_LINUX')
expect(aggregate['runs-on']).not.toContain('DSH_CI_FAILOVER_WINDOWS')
expect(aggregate['runs-on']).toContain('vm-backup')
})
it('exempts push from cancellation, so one master merge does not cancel the running drill', () => {