From 60db79f35e686476817843f5ae1702d5d8a2b112 Mon Sep 17 00:00:00 2001 From: Ramon Roche Date: Tue, 7 Apr 2026 18:21:06 -0700 Subject: [PATCH] ci(apt): extract AWS apt mirror swap into shared script The mirror swap was duplicated across two workflows. Move it into Tools/ci/use_aws_apt_mirror.sh and call the script from each workflow after checkout but before any heavy apt work like Tools/setup/ubuntu.sh. The script no-ops outside runs-on (RUNS_ON_AWS_REGION unset), so it is safe to call from forks, self-hosted runners, or local container runs without changing behavior there. The region is read from the runs-on environment instead of being hardcoded, so future region changes only need updating where the runner is provisioned. The bootstrap 'apt install git' step keeps the default mirror because git is one package and is unlikely to hit the dep11 desync issue that broke ubuntu.sh. Signed-off-by: Ramon Roche --- .github/workflows/build_deb_package.yml | 14 ++------- .github/workflows/compile_ubuntu.yml | 14 ++------- Tools/ci/use_aws_apt_mirror.sh | 42 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 22 deletions(-) create mode 100755 Tools/ci/use_aws_apt_mirror.sh diff --git a/.github/workflows/build_deb_package.yml b/.github/workflows/build_deb_package.yml index a16ef9538a..573d4fd463 100644 --- a/.github/workflows/build_deb_package.yml +++ b/.github/workflows/build_deb_package.yml @@ -90,17 +90,6 @@ jobs: - name: Fix git in container run: | - # Switch to AWS regional mirrors. runs-on instances are in us-west-2, - # and the EC2 archive mirrors are Canonical-operated, region-local, - # and sync aggressively. The default archive.ubuntu.com round-robin - # sometimes serves out-of-sync index files mid-sync, breaking - # apt-get update. - if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then - sed -i 's|http://archive.ubuntu.com/ubuntu|http://us-west-2.ec2.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://us-west-2.ec2.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources - fi - if [ -f /etc/apt/sources.list ]; then - sed -i 's|http://archive.ubuntu.com/ubuntu|http://us-west-2.ec2.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://us-west-2.ec2.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list - fi apt-get update && apt-get install -y git git config --global --add safe.directory $(realpath .) @@ -109,6 +98,9 @@ jobs: fetch-depth: 0 fetch-tags: true + - name: Use AWS regional apt mirror + run: ./Tools/ci/use_aws_apt_mirror.sh + - name: Cache apt packages uses: actions/cache@v4 with: diff --git a/.github/workflows/compile_ubuntu.yml b/.github/workflows/compile_ubuntu.yml index 3502a19d2b..27c0d46649 100644 --- a/.github/workflows/compile_ubuntu.yml +++ b/.github/workflows/compile_ubuntu.yml @@ -38,17 +38,6 @@ jobs: - name: Fix git in container run: | - # Switch to AWS regional mirrors. runs-on instances are in us-west-2, - # and the EC2 archive mirrors are Canonical-operated, region-local, - # and sync aggressively. The default archive.ubuntu.com round-robin - # sometimes serves out-of-sync index files mid-sync, breaking - # apt-get update. - if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then - sed -i 's|http://archive.ubuntu.com/ubuntu|http://us-west-2.ec2.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://us-west-2.ec2.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources - fi - if [ -f /etc/apt/sources.list ]; then - sed -i 's|http://archive.ubuntu.com/ubuntu|http://us-west-2.ec2.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://us-west-2.ec2.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list - fi # we only need this because we are running the job in a container # when checkout pulls git it does it in a shared volume # and file ownership changes between steps @@ -60,6 +49,9 @@ jobs: - uses: actions/checkout@v4 + - name: Use AWS regional apt mirror + run: ./Tools/ci/use_aws_apt_mirror.sh + - name: Install Deps, Build, and Make Quick Check run: | # we need to install dependencies and build on the same step diff --git a/Tools/ci/use_aws_apt_mirror.sh b/Tools/ci/use_aws_apt_mirror.sh new file mode 100755 index 0000000000..ea21e9aaaf --- /dev/null +++ b/Tools/ci/use_aws_apt_mirror.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Rewrite the container's apt sources to point at the AWS regional Ubuntu +# mirror that is local to the runs-on instance. +# +# The default archive.ubuntu.com round-robin sometimes serves out-of-sync +# index files mid-sync, breaking apt-get update with errors like: +# File has unexpected size (25378 != 25381). Mirror sync in progress? +# The Canonical-operated EC2 mirrors are region-local and sync aggressively, +# eliminating that failure mode. +# +# This script is a no-op outside runs-on, so it is safe to call from any CI +# job (forks, self-hosted runners, local docker runs, etc.) without changing +# behavior there. +# +# Usage (from a workflow step running inside the container): +# ./Tools/ci/use_aws_apt_mirror.sh + +set -e + +if [ -z "$RUNS_ON_AWS_REGION" ]; then + echo "use_aws_apt_mirror: not running on runs-on (RUNS_ON_AWS_REGION unset), skipping" + exit 0 +fi + +MIRROR="http://${RUNS_ON_AWS_REGION}.ec2.archive.ubuntu.com/ubuntu" +echo "use_aws_apt_mirror: rewriting apt sources to ${MIRROR}" + +# Noble (24.04+) uses the deb822 format at /etc/apt/sources.list.d/ubuntu.sources +if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then + sed -i \ + -e "s|http://archive.ubuntu.com/ubuntu|${MIRROR}|g" \ + -e "s|http://security.ubuntu.com/ubuntu|${MIRROR}|g" \ + /etc/apt/sources.list.d/ubuntu.sources +fi + +# Jammy (22.04) and earlier use the legacy /etc/apt/sources.list +if [ -f /etc/apt/sources.list ]; then + sed -i \ + -e "s|http://archive.ubuntu.com/ubuntu|${MIRROR}|g" \ + -e "s|http://security.ubuntu.com/ubuntu|${MIRROR}|g" \ + /etc/apt/sources.list +fi