#!/usr/bin/env python3 from itertools import combinations, chain from enum import Enum, auto LINUX = 'ubuntu-24.04' OSX = 'macos-latest' WINDOWS = 'windows-latest' FREEBSD = 'freebsd' AMD64 = 'amd64' ARM64 = 'arm64' PPC64LE = 'ppc64le' GITHUB_ACTIONS_TEMPLATE = """\ # This config file is generated by ./scripts/gen_gh_actions.py. # Do not edit by hand. name: {name} on: push: branches: [ dev, ci_travis ] pull_request: branches: [ dev ] jobs: {jobs} """ class Option(object): class Type: COMPILER = auto() COMPILER_FLAG = auto() CONFIGURE_FLAG = auto() MALLOC_CONF = auto() FEATURE = auto() def __init__(self, type, value): self.type = type self.value = value @staticmethod def as_compiler(value): return Option(Option.Type.COMPILER, value) @staticmethod def as_compiler_flag(value): return Option(Option.Type.COMPILER_FLAG, value) @staticmethod def as_configure_flag(value): return Option(Option.Type.CONFIGURE_FLAG, value) @staticmethod def as_malloc_conf(value): return Option(Option.Type.MALLOC_CONF, value) @staticmethod def as_feature(value): return Option(Option.Type.FEATURE, value) def __eq__(self, obj): return (isinstance(obj, Option) and obj.type == self.type and obj.value == self.value) def __repr__(self): type_names = { Option.Type.COMPILER: 'COMPILER', Option.Type.COMPILER_FLAG: 'COMPILER_FLAG', Option.Type.CONFIGURE_FLAG: 'CONFIGURE_FLAG', Option.Type.MALLOC_CONF: 'MALLOC_CONF', Option.Type.FEATURE: 'FEATURE' } return f"Option({type_names[self.type]}, {repr(self.value)})" # The 'default' configuration is gcc, on linux, with no compiler or configure # flags. We also test with clang, -m32, --enable-debug, --enable-prof, # --disable-stats, and --with-malloc-conf=tcache:false. To avoid abusing # CI resources though, we don't test all 2**7 = 128 possible combinations of these; # instead, we only test combinations of up to 2 'unusual' settings, under the # hope that bugs involving interactions of such settings are rare. MAX_UNUSUAL_OPTIONS = 2 GCC = Option.as_compiler('CC=gcc CXX=g++') CLANG = Option.as_compiler('CC=clang CXX=clang++') CL = Option.as_compiler('CC=cl.exe CXX=cl.exe') compilers_unusual = [CLANG,] CROSS_COMPILE_32BIT = Option.as_feature('CROSS_COMPILE_32BIT') feature_unusuals = [CROSS_COMPILE_32BIT] configure_flag_unusuals = [Option.as_configure_flag(opt) for opt in ( '--enable-debug', '--enable-prof', '--disable-stats', '--disable-libdl', '--enable-opt-safety-checks', '--with-lg-page=16', '--with-lg-page=16 --with-lg-hugepage=29', )] LARGE_HUGEPAGE = Option.as_configure_flag("--with-lg-page=16 --with-lg-hugepage=29") malloc_conf_unusuals = [Option.as_malloc_conf(opt) for opt in ( 'tcache:false', 'dss:primary', 'percpu_arena:percpu', 'background_thread:true', )] all_unusuals = (compilers_unusual + feature_unusuals + configure_flag_unusuals + malloc_conf_unusuals) def get_extra_cflags(os, compiler): if os == WINDOWS: # For non-CL compilers under Windows (for now it's only MinGW-GCC), # -fcommon needs to be specified to correctly handle multiple # 'malloc_conf' symbols and such, which are declared weak under Linux. # Weak symbols don't work with MinGW-GCC. if compiler != CL.value: return ['-fcommon'] else: return [] # We get some spurious errors when -Warray-bounds is enabled. extra_cflags = ['-Werror', '-Wno-array-bounds'] if compiler == CLANG.value or os == OSX: extra_cflags += [ '-Wno-unknown-warning-option', '-Wno-ignored-attributes' ] if os == OSX: extra_cflags += [ '-Wno-deprecated-declarations', ] return extra_cflags def format_env_dict(os, arch, combination): """Format environment variables as a dictionary for the matrix.""" compilers = [x.value for x in combination if x.type == Option.Type.COMPILER] compiler_flags = [x.value for x in combination if x.type == Option.Type.COMPILER_FLAG] configure_flags = [x.value for x in combination if x.type == Option.Type.CONFIGURE_FLAG] malloc_conf = [x.value for x in combination if x.type == Option.Type.MALLOC_CONF] features = [x.value for x in combination if x.type == Option.Type.FEATURE] if len(malloc_conf) > 0: configure_flags.append('--with-malloc-conf=' + ','.join(malloc_conf)) if not compilers: compiler = GCC.value else: compiler = compilers[0] cross_compile = CROSS_COMPILE_32BIT.value in features if os == LINUX and cross_compile: compiler_flags.append('-m32') env_dict = {} # Parse compiler cc_parts = compiler.split() for part in cc_parts: if part.startswith('CC='): env_dict['CC'] = part.split('=')[1] elif part.startswith('CXX='): env_dict['CXX'] = part.split('=')[1] # Add features for feature in features: env_dict[feature] = 'yes' # Add flags if compiler_flags: env_dict['COMPILER_FLAGS'] = ' '.join(compiler_flags) if configure_flags: env_dict['CONFIGURE_FLAGS'] = ' '.join(configure_flags) extra_cflags = get_extra_cflags(os, compiler) if extra_cflags: env_dict['EXTRA_CFLAGS'] = ' '.join(extra_cflags) return env_dict def generate_job_matrix_entries(os, arch, exclude, max_unusual_opts, unusuals=all_unusuals): """Generate matrix entries for a job.""" entries = [] for combination in chain.from_iterable( [combinations(unusuals, i) for i in range(max_unusual_opts + 1)]): if not any(excluded in combination for excluded in exclude): env_dict = format_env_dict(os, arch, combination) entries.append(env_dict) return entries def generate_linux_job(arch): """Generate Linux job configuration.""" os = LINUX # Only generate 2 unusual options for AMD64 to reduce matrix size max_unusual_opts = MAX_UNUSUAL_OPTIONS if arch == AMD64 else 1 exclude = [] if arch == PPC64LE: # Avoid 32 bit builds and clang on PowerPC exclude = (CROSS_COMPILE_32BIT, CLANG,) if arch == ARM64: # Avoid 32 bit build on ARM64 exclude = (CROSS_COMPILE_32BIT,) if arch != ARM64: exclude += [LARGE_HUGEPAGE] linux_configure_flags = list(configure_flag_unusuals) linux_configure_flags.append(Option.as_configure_flag("--enable-prof --enable-prof-frameptr")) linux_unusuals = (compilers_unusual + feature_unusuals + linux_configure_flags + malloc_conf_unusuals) matrix_entries = generate_job_matrix_entries(os, arch, exclude, max_unusual_opts, linux_unusuals) arch_suffix = f"-{arch}" if arch != AMD64 else "" # Select appropriate runner based on architecture if arch == ARM64: runner = "ubuntu-24.04-arm" # Free ARM64 runner for public repos (Public Preview) elif arch == PPC64LE: # GitHub doesn't provide PPC runners, would need self-hosted runner = "self-hosted-ppc64le" else: # AMD64 runner = "ubuntu-24.04" # Ubuntu 24.04 LTS job = f""" test-linux{arch_suffix}: runs-on: {runner} strategy: fail-fast: false matrix: include: """ for entry in matrix_entries: job += " - env:\n" for key, value in entry.items(): # Properly escape values with special characters if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']): job += f' {key}: "{value}"\n' else: job += f" {key}: {value}\n" # Add manual job entries manual_entries = [ { 'CC': 'gcc', 'CXX': 'g++', 'CONFIGURE_FLAGS': '--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof', 'EXTRA_CFLAGS': '-Werror -Wno-array-bounds' }, { 'CC': 'gcc', 'CXX': 'g++', 'CONFIGURE_FLAGS': '--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof', 'EXTRA_CFLAGS': '-Werror -Wno-array-bounds' } ] if arch == AMD64: for entry in manual_entries: job += " - env:\n" for key, value in entry.items(): if ' ' in str(value): job += f' {key}: "{value}"\n' else: job += f" {key}: {value}\n" job += f""" steps: - uses: actions/checkout@v4 - name: Show OS version run: | echo "=== System Information ===" uname -a echo "" echo "=== Architecture ===" uname -m arch echo "" echo "=== OS Release ===" cat /etc/os-release || true echo "" echo "=== CPU Info ===" lscpu | grep -E "Architecture|CPU op-mode|Byte Order|CPU\(s\):" || true - name: Install dependencies (32-bit) if: matrix.env.CROSS_COMPILE_32BIT == 'yes' run: | sudo dpkg --add-architecture i386 sudo apt-get update sudo apt-get install -y gcc-multilib g++-multilib libc6-dev-i386 - name: Build and test env: CC: ${{{{ matrix.env.CC }}}} CXX: ${{{{ matrix.env.CXX }}}} COMPILER_FLAGS: ${{{{ matrix.env.COMPILER_FLAGS }}}} CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}} EXTRA_CFLAGS: ${{{{ matrix.env.EXTRA_CFLAGS }}}} run: | # Verify the script generates the same output ./scripts/gen_gh_actions.py > gh_actions_script.yml # Run autoconf autoconf # Configure with flags if [ -n "$COMPILER_FLAGS" ]; then ./configure CC="${{CC}} ${{COMPILER_FLAGS}}" CXX="${{CXX}} ${{COMPILER_FLAGS}}" $CONFIGURE_FLAGS else ./configure $CONFIGURE_FLAGS fi # Build make -j3 make -j3 tests # Run tests make check """ return job def generate_macos_job(arch): """Generate macOS job configuration.""" os = OSX max_unusual_opts = 1 exclude = ([Option.as_malloc_conf(opt) for opt in ( 'dss:primary', 'background_thread:true')] + [Option.as_configure_flag('--enable-prof')] + [CLANG,]) if arch != ARM64: exclude += [LARGE_HUGEPAGE] matrix_entries = generate_job_matrix_entries(os, arch, exclude, max_unusual_opts) arch_suffix = f"-{arch}" if arch != AMD64 else "" # Select appropriate runner based on architecture # Pin both for more control over OS upgrades if arch == ARM64: runner = "macos-15" # Pinned macOS 15 on Apple Silicon else: # AMD64 runner = "macos-15-intel" # Pinned macOS 15 on Intel (last Intel runner, EOL Aug 2027) job = f""" test-macos{arch_suffix}: runs-on: {runner} strategy: fail-fast: false matrix: include: """ for entry in matrix_entries: job += " - env:\n" for key, value in entry.items(): if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']): job += f' {key}: "{value}"\n' else: job += f" {key}: {value}\n" job += f""" steps: - uses: actions/checkout@v4 - name: Show OS version run: | echo "=== macOS Version ===" sw_vers echo "" echo "=== Architecture ===" uname -m arch echo "" echo "=== CPU Info ===" sysctl -n machdep.cpu.brand_string sysctl -n hw.machine - name: Install dependencies run: | brew install autoconf - name: Build and test env: CC: ${{{{ matrix.env.CC || 'gcc' }}}} CXX: ${{{{ matrix.env.CXX || 'g++' }}}} COMPILER_FLAGS: ${{{{ matrix.env.COMPILER_FLAGS }}}} CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}} EXTRA_CFLAGS: ${{{{ matrix.env.EXTRA_CFLAGS }}}} run: | # Run autoconf autoconf # Configure with flags if [ -n "$COMPILER_FLAGS" ]; then ./configure CC="${{CC}} ${{COMPILER_FLAGS}}" CXX="${{CXX}} ${{COMPILER_FLAGS}}" $CONFIGURE_FLAGS else ./configure $CONFIGURE_FLAGS fi # Build make -j3 make -j3 tests # Run tests make check """ return job def generate_windows_job(arch): """Generate Windows job configuration.""" os = WINDOWS max_unusual_opts = 3 unusuals = ( Option.as_configure_flag('--enable-debug'), CL, CROSS_COMPILE_32BIT, ) matrix_entries = generate_job_matrix_entries(os, arch, (), max_unusual_opts, unusuals) arch_suffix = f"-{arch}" if arch != AMD64 else "" # Use latest for Windows - tends to be backward compatible and stable job = f""" test-windows{arch_suffix}: runs-on: windows-latest strategy: fail-fast: false matrix: include: """ for entry in matrix_entries: job += " - env:\n" for key, value in entry.items(): if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']): job += f' {key}: "{value}"\n' else: job += f" {key}: {value}\n" job += f""" steps: - uses: actions/checkout@v4 - name: Show OS version shell: cmd run: | echo === Windows Version === systeminfo | findstr /B /C:"OS Name" /C:"OS Version" ver echo. echo === Architecture === echo PROCESSOR_ARCHITECTURE=%PROCESSOR_ARCHITECTURE% echo. - name: Setup MSYS2 uses: msys2/setup-msys2@v2 with: msystem: ${{{{ matrix.env.CROSS_COMPILE_32BIT == 'yes' && 'MINGW32' || 'MINGW64' }}}} update: true install: >- autotools git pacboy: >- make:p gcc:p binutils:p - name: Build and test (MinGW-GCC) if: matrix.env.CC != 'cl.exe' shell: msys2 {{0}} env: CC: ${{{{ matrix.env.CC || 'gcc' }}}} CXX: ${{{{ matrix.env.CXX || 'g++' }}}} COMPILER_FLAGS: ${{{{ matrix.env.COMPILER_FLAGS }}}} CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}} EXTRA_CFLAGS: ${{{{ matrix.env.EXTRA_CFLAGS }}}} run: | # Run autoconf autoconf # Configure with flags if [ -n "$COMPILER_FLAGS" ]; then ./configure CC="${{CC}} ${{COMPILER_FLAGS}}" CXX="${{CXX}} ${{COMPILER_FLAGS}}" $CONFIGURE_FLAGS else ./configure $CONFIGURE_FLAGS fi # Build (mingw32-make is the "make" command in MSYS2) mingw32-make -j3 mingw32-make tests # Run tests mingw32-make -k check - name: Setup MSVC environment if: matrix.env.CC == 'cl.exe' uses: ilammy/msvc-dev-cmd@v1 with: arch: ${{{{ matrix.env.CROSS_COMPILE_32BIT == 'yes' && 'x86' || 'x64' }}}} - name: Build and test (MSVC) if: matrix.env.CC == 'cl.exe' shell: msys2 {{0}} env: CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}} MSYS2_PATH_TYPE: inherit run: | # Export MSVC environment variables for configure export CC=cl.exe export CXX=cl.exe export AR=lib.exe export NM=dumpbin.exe export RANLIB=: # Verify cl.exe is accessible (should be in PATH via inherit) if ! which cl.exe > /dev/null 2>&1; then echo "cl.exe not found, trying to locate MSVC..." # Find and add MSVC bin directory to PATH MSVC_BIN=$(cmd.exe /c "echo %VCToolsInstallDir%" | tr -d '\\\\r' | sed 's/\\\\\\\\\\\\\\\\/\\//g' | sed 's/C:/\\\\/c/g') if [ -n "$MSVC_BIN" ]; then export PATH="$PATH:$MSVC_BIN/bin/Hostx64/x64:$MSVC_BIN/bin/Hostx86/x86" fi fi # Run autoconf autoconf # Configure with MSVC ./configure CC=cl.exe CXX=cl.exe AR=lib.exe $CONFIGURE_FLAGS # Build (mingw32-make is the "make" command in MSYS2) mingw32-make -j3 # Build tests sequentially due to PDB file issues mingw32-make tests # Run tests mingw32-make -k check """ return job def generate_freebsd_job(arch): """Generate FreeBSD job configuration.""" # FreeBSD runs in a VM on ubuntu-latest, not native job = f""" test-freebsd: runs-on: ubuntu-latest strategy: fail-fast: false matrix: debug: ['--enable-debug', '--disable-debug'] prof: ['--enable-prof', '--disable-prof'] arch: ['64-bit', '32-bit'] uncommon: - '' - '--with-lg-page=16 --with-malloc-conf=tcache:false' name: FreeBSD (${{{{ matrix.arch }}}}, debug=${{{{ matrix.debug }}}}, prof=${{{{ matrix.prof }}}}${{{{ matrix.uncommon && ', uncommon' || '' }}}}) steps: - uses: actions/checkout@v4 with: fetch-depth: 1 - name: Test on FreeBSD uses: vmactions/freebsd-vm@v1 with: release: '15.0' usesh: true prepare: | pkg install -y autoconf gmake run: | # Verify we're running in FreeBSD echo "==== System Information ====" uname -a freebsd-version echo "============================" # Set compiler flags for 32-bit if needed if [ "${{{{ matrix.arch }}}}" = "32-bit" ]; then export CC="cc -m32" export CXX="c++ -m32" fi # Generate configure script autoconf # Configure with matrix options ./configure --with-jemalloc-prefix=ci_ ${{{{ matrix.debug }}}} ${{{{ matrix.prof }}}} ${{{{ matrix.uncommon }}}} # Get CPU count for parallel builds export JFLAG=$(sysctl -n kern.smp.cpus) gmake -j${{JFLAG}} gmake -j${{JFLAG}} tests gmake check """ return job def main(): import sys # Determine which workflow to generate based on command-line argument workflow_type = sys.argv[1] if len(sys.argv) > 1 else 'linux' if workflow_type == 'linux': jobs = '\n'.join(( generate_linux_job(AMD64), generate_linux_job(ARM64), )) print(GITHUB_ACTIONS_TEMPLATE.format(name='Linux CI', jobs=jobs)) elif workflow_type == 'macos': jobs = '\n'.join(( generate_macos_job(AMD64), # Intel x86_64 generate_macos_job(ARM64), # Apple Silicon )) print(GITHUB_ACTIONS_TEMPLATE.format(name='macOS CI', jobs=jobs)) elif workflow_type == 'windows': jobs = generate_windows_job(AMD64) print(GITHUB_ACTIONS_TEMPLATE.format(name='Windows CI', jobs=jobs)) elif workflow_type == 'freebsd': jobs = generate_freebsd_job(AMD64) print(GITHUB_ACTIONS_TEMPLATE.format(name='FreeBSD CI', jobs=jobs)) elif workflow_type == 'all': # Generate all workflow files linux_jobs = '\n'.join(( generate_linux_job(AMD64), generate_linux_job(ARM64), )) macos_jobs = '\n'.join(( generate_macos_job(AMD64), # Intel generate_macos_job(ARM64), # Apple Silicon )) windows_jobs = generate_windows_job(AMD64) freebsd_jobs = generate_freebsd_job(AMD64) all_jobs = '\n'.join((linux_jobs, macos_jobs, windows_jobs, freebsd_jobs)) print(GITHUB_ACTIONS_TEMPLATE.format(name='CI', jobs=all_jobs)) else: print(f"Unknown workflow type: {workflow_type}", file=sys.stderr) print("Usage: gen_gh_actions.py [linux|macos|windows|freebsd|all]", file=sys.stderr) sys.exit(1) if __name__ == '__main__': main()