jemalloc/scripts/gen_gh_actions.py

#!/usr/bin/env python3

from itertools import combinations, chain
from enum import Enum, auto


LINUX = 'ubuntu-24.04'
OSX = 'macos-latest'
WINDOWS = 'windows-latest'
FREEBSD = 'freebsd'

AMD64 = 'amd64'
ARM64 = 'arm64'
PPC64LE = 'ppc64le'


GITHUB_ACTIONS_TEMPLATE = """\
# This config file is generated by ./scripts/gen_gh_actions.py.
# Do not edit by hand.

name: {name}

on:
  push:
    branches: [ dev, ci_travis ]
  pull_request:
    branches: [ dev ]

jobs:
{jobs}
"""


class Option(object):
    class Type:
        COMPILER = auto()
        COMPILER_FLAG = auto()
        CONFIGURE_FLAG = auto()
        MALLOC_CONF = auto()
        FEATURE = auto()

    def __init__(self, type, value):
        self.type = type
        self.value = value

    @staticmethod
    def as_compiler(value):
        return Option(Option.Type.COMPILER, value)

    @staticmethod
    def as_compiler_flag(value):
        return Option(Option.Type.COMPILER_FLAG, value)

    @staticmethod
    def as_configure_flag(value):
        return Option(Option.Type.CONFIGURE_FLAG, value)

    @staticmethod
    def as_malloc_conf(value):
        return Option(Option.Type.MALLOC_CONF, value)

    @staticmethod
    def as_feature(value):
        return Option(Option.Type.FEATURE, value)

    def __eq__(self, obj):
        return (isinstance(obj, Option) and obj.type == self.type
                and obj.value == self.value)

    def __repr__(self):
        type_names = {
            Option.Type.COMPILER: 'COMPILER',
            Option.Type.COMPILER_FLAG: 'COMPILER_FLAG',
            Option.Type.CONFIGURE_FLAG: 'CONFIGURE_FLAG',
            Option.Type.MALLOC_CONF: 'MALLOC_CONF',
            Option.Type.FEATURE: 'FEATURE'
        }
        return f"Option({type_names[self.type]}, {repr(self.value)})"


# The 'default' configuration is gcc, on linux, with no compiler or configure
# flags.  We also test with clang, -m32, --enable-debug, --enable-prof,
# --disable-stats, and --with-malloc-conf=tcache:false.  To avoid abusing
# CI resources though, we don't test all 2**7 = 128 possible combinations of these;
# instead, we only test combinations of up to 2 'unusual' settings, under the
# hope that bugs involving interactions of such settings are rare.
MAX_UNUSUAL_OPTIONS = 2


GCC = Option.as_compiler('CC=gcc CXX=g++')
CLANG = Option.as_compiler('CC=clang CXX=clang++')
CL = Option.as_compiler('CC=cl.exe CXX=cl.exe')


compilers_unusual = [CLANG,]


CROSS_COMPILE_32BIT = Option.as_feature('CROSS_COMPILE_32BIT')
feature_unusuals = [CROSS_COMPILE_32BIT]


configure_flag_unusuals = [Option.as_configure_flag(opt) for opt in (
    '--enable-debug',
    '--enable-prof',
    '--disable-stats',
    '--disable-libdl',
    '--enable-opt-safety-checks',
    '--with-lg-page=16',
    '--with-lg-page=16 --with-lg-hugepage=29',
)]
LARGE_HUGEPAGE = Option.as_configure_flag("--with-lg-page=16 --with-lg-hugepage=29")


malloc_conf_unusuals = [Option.as_malloc_conf(opt) for opt in (
    'tcache:false',
    'dss:primary',
    'percpu_arena:percpu',
    'background_thread:true',
)]


all_unusuals = (compilers_unusual + feature_unusuals
    + configure_flag_unusuals + malloc_conf_unusuals)


def get_extra_cflags(os, compiler):
    if os == WINDOWS:
        # For non-CL compilers under Windows (for now it's only MinGW-GCC),
        # -fcommon needs to be specified to correctly handle multiple
        # 'malloc_conf' symbols and such, which are declared weak under Linux.
        # Weak symbols don't work with MinGW-GCC.
        if compiler != CL.value:
            return ['-fcommon']
        else:
            return []

    # We get some spurious errors when -Warray-bounds is enabled.
    extra_cflags = ['-Werror', '-Wno-array-bounds']
    if compiler == CLANG.value or os == OSX:
        extra_cflags += [
            '-Wno-unknown-warning-option',
            '-Wno-ignored-attributes'
        ]
    if os == OSX:
        extra_cflags += [
            '-Wno-deprecated-declarations',
        ]
    return extra_cflags


def format_env_dict(os, arch, combination):
    """Format environment variables as a dictionary for the matrix."""
    compilers = [x.value for x in combination if x.type == Option.Type.COMPILER]
    compiler_flags = [x.value for x in combination if x.type == Option.Type.COMPILER_FLAG]
    configure_flags = [x.value for x in combination if x.type == Option.Type.CONFIGURE_FLAG]
    malloc_conf = [x.value for x in combination if x.type == Option.Type.MALLOC_CONF]
    features = [x.value for x in combination if x.type == Option.Type.FEATURE]

    if len(malloc_conf) > 0:
        configure_flags.append('--with-malloc-conf=' + ','.join(malloc_conf))

    if not compilers:
        compiler = GCC.value
    else:
        compiler = compilers[0]

    cross_compile = CROSS_COMPILE_32BIT.value in features
    if os == LINUX and cross_compile:
        compiler_flags.append('-m32')

    env_dict = {}

    # Parse compiler
    cc_parts = compiler.split()
    for part in cc_parts:
        if part.startswith('CC='):
            env_dict['CC'] = part.split('=')[1]
        elif part.startswith('CXX='):
            env_dict['CXX'] = part.split('=')[1]

    # Add features
    for feature in features:
        env_dict[feature] = 'yes'

    # Add flags
    if compiler_flags:
        env_dict['COMPILER_FLAGS'] = ' '.join(compiler_flags)
    if configure_flags:
        env_dict['CONFIGURE_FLAGS'] = ' '.join(configure_flags)

    extra_cflags = get_extra_cflags(os, compiler)
    if extra_cflags:
        env_dict['EXTRA_CFLAGS'] = ' '.join(extra_cflags)

    return env_dict


def generate_job_matrix_entries(os, arch, exclude, max_unusual_opts, unusuals=all_unusuals):
    """Generate matrix entries for a job."""
    entries = []
    for combination in chain.from_iterable(
            [combinations(unusuals, i) for i in range(max_unusual_opts + 1)]):
        if not any(excluded in combination for excluded in exclude):
            env_dict = format_env_dict(os, arch, combination)
            entries.append(env_dict)
    return entries


def generate_linux_job(arch):
    """Generate Linux job configuration."""
    os = LINUX

    # Only generate 2 unusual options for AMD64 to reduce matrix size
    max_unusual_opts = MAX_UNUSUAL_OPTIONS if arch == AMD64 else 1

    exclude = []
    if arch == PPC64LE:
        # Avoid 32 bit builds and clang on PowerPC
        exclude = (CROSS_COMPILE_32BIT, CLANG,)
    if arch == ARM64:
        # Avoid 32 bit build on ARM64
        exclude = (CROSS_COMPILE_32BIT,)

    if arch != ARM64:
        exclude += [LARGE_HUGEPAGE]

    linux_configure_flags = list(configure_flag_unusuals)
    linux_configure_flags.append(Option.as_configure_flag("--enable-prof --enable-prof-frameptr"))

    linux_unusuals = (compilers_unusual + feature_unusuals
                    + linux_configure_flags + malloc_conf_unusuals)

    matrix_entries = generate_job_matrix_entries(os, arch, exclude, max_unusual_opts, linux_unusuals)

    arch_suffix = f"-{arch}" if arch != AMD64 else ""

    # Select appropriate runner based on architecture
    if arch == ARM64:
        runner = "ubuntu-24.04-arm"    # Free ARM64 runner for public repos (Public Preview)
    elif arch == PPC64LE:
        # GitHub doesn't provide PPC runners, would need self-hosted
        runner = "self-hosted-ppc64le"
    else:  # AMD64
        runner = "ubuntu-24.04"        # Ubuntu 24.04 LTS

    job = f"""  test-linux{arch_suffix}:
    runs-on: {runner}
    strategy:
      fail-fast: false
      matrix:
        include:
"""

    for entry in matrix_entries:
        job += "          - env:\n"
        for key, value in entry.items():
            # Properly escape values with special characters
            if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
                job += f'              {key}: "{value}"\n'
            else:
                job += f"              {key}: {value}\n"

    # Add manual job entries
    manual_entries = [
        {
            'CC': 'gcc',
            'CXX': 'g++',
            'CONFIGURE_FLAGS': '--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof',
            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
        },
        {
            'CC': 'gcc',
            'CXX': 'g++',
            'CONFIGURE_FLAGS': '--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof',
            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
        }
    ]

    if arch == AMD64:
        for entry in manual_entries:
            job += "          - env:\n"
            for key, value in entry.items():
                if ' ' in str(value):
                    job += f'              {key}: "{value}"\n'
                else:
                    job += f"              {key}: {value}\n"

    job += f"""
    steps:
    - uses: actions/checkout@v4

    - name: Show OS version
      run: |
        echo "=== System Information ==="
        uname -a
        echo ""
        echo "=== Architecture ==="
        uname -m
        arch
        echo ""
        echo "=== OS Release ==="
        cat /etc/os-release || true
        echo ""
        echo "=== CPU Info ==="
        lscpu | grep -E "Architecture|CPU op-mode|Byte Order|CPU\(s\):" || true

    - name: Install dependencies (32-bit)
      if: matrix.env.CROSS_COMPILE_32BIT == 'yes'
      run: |
        sudo dpkg --add-architecture i386
        sudo apt-get update
        sudo apt-get install -y gcc-multilib g++-multilib libc6-dev-i386

    - name: Build and test
      env:
        CC: ${{{{ matrix.env.CC }}}}
        CXX: ${{{{ matrix.env.CXX }}}}
        COMPILER_FLAGS: ${{{{ matrix.env.COMPILER_FLAGS }}}}
        CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}}
        EXTRA_CFLAGS: ${{{{ matrix.env.EXTRA_CFLAGS }}}}
      run: |
        # Verify the script generates the same output
        ./scripts/gen_gh_actions.py > gh_actions_script.yml

        # Run autoconf
        autoconf

        # Configure with flags
        if [ -n "$COMPILER_FLAGS" ]; then
          ./configure CC="${{CC}} ${{COMPILER_FLAGS}}" CXX="${{CXX}} ${{COMPILER_FLAGS}}" $CONFIGURE_FLAGS
        else
          ./configure $CONFIGURE_FLAGS
        fi

        # Build
        make -j3
        make -j3 tests

        # Run tests
        make check

"""

    return job


def generate_macos_job(arch):
    """Generate macOS job configuration."""
    os = OSX
    max_unusual_opts = 1

    exclude = ([Option.as_malloc_conf(opt) for opt in (
            'dss:primary',
            'background_thread:true')] +
        [Option.as_configure_flag('--enable-prof')] +
        [CLANG,])

    if arch != ARM64:
        exclude += [LARGE_HUGEPAGE]

    matrix_entries = generate_job_matrix_entries(os, arch, exclude, max_unusual_opts)

    arch_suffix = f"-{arch}" if arch != AMD64 else ""

    # Select appropriate runner based on architecture
    # Pin both for more control over OS upgrades
    if arch == ARM64:
        runner = "macos-15"          # Pinned macOS 15 on Apple Silicon
    else:  # AMD64
        runner = "macos-15-intel"    # Pinned macOS 15 on Intel (last Intel runner, EOL Aug 2027)

    job = f"""  test-macos{arch_suffix}:
    runs-on: {runner}
    strategy:
      fail-fast: false
      matrix:
        include:
"""

    for entry in matrix_entries:
        job += "          - env:\n"
        for key, value in entry.items():
            if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
                job += f'              {key}: "{value}"\n'
            else:
                job += f"              {key}: {value}\n"

    job += f"""
    steps:
    - uses: actions/checkout@v4

    - name: Show OS version
      run: |
        echo "=== macOS Version ==="
        sw_vers
        echo ""
        echo "=== Architecture ==="
        uname -m
        arch
        echo ""
        echo "=== CPU Info ==="
        sysctl -n machdep.cpu.brand_string
        sysctl -n hw.machine

    - name: Install dependencies
      run: |
        brew install autoconf

    - name: Build and test
      env:
        CC: ${{{{ matrix.env.CC || 'gcc' }}}}
        CXX: ${{{{ matrix.env.CXX || 'g++' }}}}
        COMPILER_FLAGS: ${{{{ matrix.env.COMPILER_FLAGS }}}}
        CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}}
        EXTRA_CFLAGS: ${{{{ matrix.env.EXTRA_CFLAGS }}}}
      run: |
        # Run autoconf
        autoconf

        # Configure with flags
        if [ -n "$COMPILER_FLAGS" ]; then
          ./configure CC="${{CC}} ${{COMPILER_FLAGS}}" CXX="${{CXX}} ${{COMPILER_FLAGS}}" $CONFIGURE_FLAGS
        else
          ./configure $CONFIGURE_FLAGS
        fi

        # Build
        make -j3
        make -j3 tests

        # Run tests
        make check

"""

    return job


def generate_windows_job(arch):
    """Generate Windows job configuration."""
    os = WINDOWS
    max_unusual_opts = 3
    unusuals = (
        Option.as_configure_flag('--enable-debug'),
        CL,
        CROSS_COMPILE_32BIT,
    )

    matrix_entries = generate_job_matrix_entries(os, arch, (), max_unusual_opts, unusuals)

    arch_suffix = f"-{arch}" if arch != AMD64 else ""

    # Use latest for Windows - tends to be backward compatible and stable
    job = f"""  test-windows{arch_suffix}:
    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        include:
"""

    for entry in matrix_entries:
        job += "          - env:\n"
        for key, value in entry.items():
            if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
                job += f'              {key}: "{value}"\n'
            else:
                job += f"              {key}: {value}\n"

    job += f"""
    steps:
    - uses: actions/checkout@v4

    - name: Show OS version
      shell: cmd
      run: |
        echo === Windows Version ===
        systeminfo | findstr /B /C:"OS Name" /C:"OS Version"
        ver
        echo.
        echo === Architecture ===
        echo PROCESSOR_ARCHITECTURE=%PROCESSOR_ARCHITECTURE%
        echo.

    - name: Setup MSYS2
      uses: msys2/setup-msys2@v2
      with:
        msystem: ${{{{ matrix.env.CROSS_COMPILE_32BIT == 'yes' && 'MINGW32' || 'MINGW64' }}}}
        update: true
        install: >-
          autotools
          git
        pacboy: >-
          make:p
          gcc:p
          binutils:p

    - name: Build and test (MinGW-GCC)
      if: matrix.env.CC != 'cl.exe'
      shell: msys2 {{0}}
      env:
        CC: ${{{{ matrix.env.CC || 'gcc' }}}}
        CXX: ${{{{ matrix.env.CXX || 'g++' }}}}
        COMPILER_FLAGS: ${{{{ matrix.env.COMPILER_FLAGS }}}}
        CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}}
        EXTRA_CFLAGS: ${{{{ matrix.env.EXTRA_CFLAGS }}}}
      run: |
        # Run autoconf
        autoconf

        # Configure with flags
        if [ -n "$COMPILER_FLAGS" ]; then
          ./configure CC="${{CC}} ${{COMPILER_FLAGS}}" CXX="${{CXX}} ${{COMPILER_FLAGS}}" $CONFIGURE_FLAGS
        else
          ./configure $CONFIGURE_FLAGS
        fi

        # Build (mingw32-make is the "make" command in MSYS2)
        mingw32-make -j3
        mingw32-make tests

        # Run tests
        mingw32-make -k check

    - name: Setup MSVC environment
      if: matrix.env.CC == 'cl.exe'
      uses: ilammy/msvc-dev-cmd@v1
      with:
        arch: ${{{{ matrix.env.CROSS_COMPILE_32BIT == 'yes' && 'x86' || 'x64' }}}}

    - name: Build and test (MSVC)
      if: matrix.env.CC == 'cl.exe'
      shell: msys2 {{0}}
      env:
        CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}}
        MSYS2_PATH_TYPE: inherit
      run: |
        # Export MSVC environment variables for configure
        export CC=cl.exe
        export CXX=cl.exe
        export AR=lib.exe
        export NM=dumpbin.exe
        export RANLIB=:

        # Verify cl.exe is accessible (should be in PATH via inherit)
        if ! which cl.exe > /dev/null 2>&1; then
          echo "cl.exe not found, trying to locate MSVC..."
          # Find and add MSVC bin directory to PATH
          MSVC_BIN=$(cmd.exe /c "echo %VCToolsInstallDir%" | tr -d '\\\\r' | sed 's/\\\\\\\\\\\\\\\\/\\//g' | sed 's/C:/\\\\/c/g')
          if [ -n "$MSVC_BIN" ]; then
            export PATH="$PATH:$MSVC_BIN/bin/Hostx64/x64:$MSVC_BIN/bin/Hostx86/x86"
          fi
        fi

        # Run autoconf
        autoconf

        # Configure with MSVC
        ./configure CC=cl.exe CXX=cl.exe AR=lib.exe $CONFIGURE_FLAGS

        # Build (mingw32-make is the "make" command in MSYS2)
        mingw32-make -j3
        # Build tests sequentially due to PDB file issues
        mingw32-make tests

        # Run tests
        mingw32-make -k check

"""

    return job


def generate_freebsd_job(arch):
    """Generate FreeBSD job configuration."""
    # FreeBSD runs in a VM on ubuntu-latest, not native

    job = f"""  test-freebsd:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        debug: ['--enable-debug', '--disable-debug']
        prof: ['--enable-prof', '--disable-prof']
        arch: ['64-bit', '32-bit']
        uncommon:
          - ''
          - '--with-lg-page=16 --with-malloc-conf=tcache:false'

    name: FreeBSD (${{{{ matrix.arch }}}}, debug=${{{{ matrix.debug }}}}, prof=${{{{ matrix.prof }}}}${{{{ matrix.uncommon && ', uncommon' || '' }}}})

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 1

    - name: Test on FreeBSD
      uses: vmactions/freebsd-vm@v1
      with:
        release: '15.0'
        usesh: true
        prepare: |
          pkg install -y autoconf gmake
        run: |
          # Verify we're running in FreeBSD
          echo "==== System Information ===="
          uname -a
          freebsd-version
          echo "============================"

          # Set compiler flags for 32-bit if needed
          if [ "${{{{ matrix.arch }}}}" = "32-bit" ]; then
            export CC="cc -m32"
            export CXX="c++ -m32"
          fi

          # Generate configure script
          autoconf

          # Configure with matrix options
          ./configure --with-jemalloc-prefix=ci_ ${{{{ matrix.debug }}}} ${{{{ matrix.prof }}}} ${{{{ matrix.uncommon }}}}

          # Get CPU count for parallel builds
          export JFLAG=$(sysctl -n kern.smp.cpus)

          gmake -j${{JFLAG}}
          gmake -j${{JFLAG}} tests
          gmake check

"""

    return job


def main():
    import sys

    # Determine which workflow to generate based on command-line argument
    workflow_type = sys.argv[1] if len(sys.argv) > 1 else 'linux'

    if workflow_type == 'linux':
        jobs = '\n'.join((
            generate_linux_job(AMD64),
            generate_linux_job(ARM64),
        ))
        print(GITHUB_ACTIONS_TEMPLATE.format(name='Linux CI', jobs=jobs))

    elif workflow_type == 'macos':
        jobs = '\n'.join((
            generate_macos_job(AMD64),   # Intel x86_64
            generate_macos_job(ARM64),   # Apple Silicon
        ))
        print(GITHUB_ACTIONS_TEMPLATE.format(name='macOS CI', jobs=jobs))

    elif workflow_type == 'windows':
        jobs = generate_windows_job(AMD64)
        print(GITHUB_ACTIONS_TEMPLATE.format(name='Windows CI', jobs=jobs))

    elif workflow_type == 'freebsd':
        jobs = generate_freebsd_job(AMD64)
        print(GITHUB_ACTIONS_TEMPLATE.format(name='FreeBSD CI', jobs=jobs))

    elif workflow_type == 'all':
        # Generate all workflow files
        linux_jobs = '\n'.join((
            generate_linux_job(AMD64),
            generate_linux_job(ARM64),
        ))
        macos_jobs = '\n'.join((
            generate_macos_job(AMD64),   # Intel
            generate_macos_job(ARM64),   # Apple Silicon
        ))
        windows_jobs = generate_windows_job(AMD64)
        freebsd_jobs = generate_freebsd_job(AMD64)

        all_jobs = '\n'.join((linux_jobs, macos_jobs, windows_jobs, freebsd_jobs))
        print(GITHUB_ACTIONS_TEMPLATE.format(name='CI', jobs=all_jobs))

    else:
        print(f"Unknown workflow type: {workflow_type}", file=sys.stderr)
        print("Usage: gen_gh_actions.py [linux|macos|windows|freebsd|all]", file=sys.stderr)
        sys.exit(1)


if __name__ == '__main__':
    main()