jemalloc/scripts/gen_gh_actions.py
Slobodan Predolac fb5499aa9c Handle jemalloc calls after TSD teardown
This only changes behavior for the tsd_generic path, where
pthread_getspecific() can report no TSD after the pthread key destructor
has finished.  Other TSD backends keep tsd_teardown_done() as a constant
false, so the added checks compile out there.

Avoid recreating TSD for late deallocations.  Preserve existing
reincarnation behavior for late allocations and nonzero reallocations.

Add Linux CI coverage for force_tls=0, with and without --enable-debug,
to exercise the Android-equivalent generic TSD path.
2026-07-09 11:30:15 -04:00

760 lines
22 KiB
Python
Executable file

#!/usr/bin/env python3
from itertools import combinations, chain
from enum import Enum, auto
LINUX = 'ubuntu-24.04'
OSX = 'macos-latest'
WINDOWS = 'windows-latest'
FREEBSD = 'freebsd'
AMD64 = 'amd64'
ARM64 = 'arm64'
PPC64LE = 'ppc64le'
GITHUB_ACTIONS_TEMPLATE = """\
# This config file is generated by ./scripts/gen_gh_actions.py.
# Do not edit by hand.
name: {name}
on:
push:
branches: [ dev, ci_travis ]
pull_request:
branches: [ dev ]
jobs:
{jobs}
"""
class Option(object):
class Type:
COMPILER = auto()
COMPILER_FLAG = auto()
CONFIGURE_FLAG = auto()
MALLOC_CONF = auto()
FEATURE = auto()
def __init__(self, type, value):
self.type = type
self.value = value
@staticmethod
def as_compiler(value):
return Option(Option.Type.COMPILER, value)
@staticmethod
def as_compiler_flag(value):
return Option(Option.Type.COMPILER_FLAG, value)
@staticmethod
def as_configure_flag(value):
return Option(Option.Type.CONFIGURE_FLAG, value)
@staticmethod
def as_malloc_conf(value):
return Option(Option.Type.MALLOC_CONF, value)
@staticmethod
def as_feature(value):
return Option(Option.Type.FEATURE, value)
def __eq__(self, obj):
return (isinstance(obj, Option) and obj.type == self.type
and obj.value == self.value)
def __repr__(self):
type_names = {
Option.Type.COMPILER: 'COMPILER',
Option.Type.COMPILER_FLAG: 'COMPILER_FLAG',
Option.Type.CONFIGURE_FLAG: 'CONFIGURE_FLAG',
Option.Type.MALLOC_CONF: 'MALLOC_CONF',
Option.Type.FEATURE: 'FEATURE'
}
return f"Option({type_names[self.type]}, {repr(self.value)})"
# The 'default' configuration is gcc, on linux, with no compiler or configure
# flags. We also test with clang, -m32, --enable-debug, --enable-prof,
# --disable-stats, and --with-malloc-conf=tcache:false. To avoid abusing
# CI resources though, we don't test all 2**7 = 128 possible combinations of these;
# instead, we only test combinations of up to 2 'unusual' settings, under the
# hope that bugs involving interactions of such settings are rare.
MAX_UNUSUAL_OPTIONS = 2
GCC = Option.as_compiler('CC=gcc CXX=g++')
CLANG = Option.as_compiler('CC=clang CXX=clang++')
CL = Option.as_compiler('CC=cl.exe CXX=cl.exe')
compilers_unusual = [CLANG,]
CROSS_COMPILE_32BIT = Option.as_feature('CROSS_COMPILE_32BIT')
feature_unusuals = [CROSS_COMPILE_32BIT]
configure_flag_unusuals = [Option.as_configure_flag(opt) for opt in (
'--enable-debug',
'--enable-prof',
'--disable-stats',
'--disable-libdl',
'--enable-opt-safety-checks',
'--with-lg-page=16',
'--with-lg-page=16 --with-lg-hugepage=29',
)]
LARGE_HUGEPAGE = Option.as_configure_flag("--with-lg-page=16 --with-lg-hugepage=29")
malloc_conf_unusuals = [Option.as_malloc_conf(opt) for opt in (
'tcache:false',
'dss:primary',
'percpu_arena:percpu',
'background_thread:true',
)]
all_unusuals = (compilers_unusual + feature_unusuals
+ configure_flag_unusuals + malloc_conf_unusuals)
def get_extra_cflags(os, compiler):
if os == WINDOWS:
# For non-CL compilers under Windows (for now it's only MinGW-GCC),
# -fcommon needs to be specified to correctly handle multiple
# 'malloc_conf' symbols and such, which are declared weak under Linux.
# Weak symbols don't work with MinGW-GCC.
if compiler != CL.value:
return ['-fcommon']
else:
return []
# We get some spurious errors when -Warray-bounds is enabled.
extra_cflags = ['-Werror', '-Wno-array-bounds']
if compiler == CLANG.value or os == OSX:
extra_cflags += [
'-Wno-unknown-warning-option',
'-Wno-ignored-attributes'
]
if os == OSX:
extra_cflags += [
'-Wno-deprecated-declarations',
]
return extra_cflags
def format_env_dict(os, arch, combination):
"""Format environment variables as a dictionary for the matrix."""
compilers = [x.value for x in combination if x.type == Option.Type.COMPILER]
compiler_flags = [x.value for x in combination if x.type == Option.Type.COMPILER_FLAG]
configure_flags = [x.value for x in combination if x.type == Option.Type.CONFIGURE_FLAG]
malloc_conf = [x.value for x in combination if x.type == Option.Type.MALLOC_CONF]
features = [x.value for x in combination if x.type == Option.Type.FEATURE]
if len(malloc_conf) > 0:
configure_flags.append('--with-malloc-conf=' + ','.join(malloc_conf))
if not compilers:
compiler = GCC.value
else:
compiler = compilers[0]
cross_compile = CROSS_COMPILE_32BIT.value in features
if os == LINUX and cross_compile:
compiler_flags.append('-m32')
env_dict = {}
# Parse compiler
cc_parts = compiler.split()
for part in cc_parts:
if part.startswith('CC='):
env_dict['CC'] = part.split('=')[1]
elif part.startswith('CXX='):
env_dict['CXX'] = part.split('=')[1]
# Add features
for feature in features:
env_dict[feature] = 'yes'
# Add flags
if compiler_flags:
env_dict['COMPILER_FLAGS'] = ' '.join(compiler_flags)
if configure_flags:
env_dict['CONFIGURE_FLAGS'] = ' '.join(configure_flags)
extra_cflags = get_extra_cflags(os, compiler)
if extra_cflags:
env_dict['EXTRA_CFLAGS'] = ' '.join(extra_cflags)
return env_dict
def generate_job_matrix_entries(os, arch, exclude, max_unusual_opts, unusuals=all_unusuals):
"""Generate matrix entries for a job."""
entries = []
for combination in chain.from_iterable(
[combinations(unusuals, i) for i in range(max_unusual_opts + 1)]):
if not any(excluded in combination for excluded in exclude):
env_dict = format_env_dict(os, arch, combination)
entries.append(env_dict)
return entries
def generate_linux_job(arch):
"""Generate Linux job configuration."""
os = LINUX
# Only generate 2 unusual options for AMD64 to reduce matrix size
max_unusual_opts = MAX_UNUSUAL_OPTIONS if arch == AMD64 else 1
exclude = []
if arch == PPC64LE:
# Avoid 32 bit builds and clang on PowerPC
exclude = (CROSS_COMPILE_32BIT, CLANG,)
if arch == ARM64:
# Avoid 32 bit build on ARM64
exclude = (CROSS_COMPILE_32BIT,)
if arch != ARM64:
exclude += [LARGE_HUGEPAGE]
linux_configure_flags = list(configure_flag_unusuals)
linux_configure_flags.append(Option.as_configure_flag("--enable-prof --enable-prof-frameptr"))
linux_unusuals = (compilers_unusual + feature_unusuals
+ linux_configure_flags + malloc_conf_unusuals)
matrix_entries = generate_job_matrix_entries(os, arch, exclude, max_unusual_opts, linux_unusuals)
arch_suffix = f"-{arch}" if arch != AMD64 else ""
# Select appropriate runner based on architecture
if arch == ARM64:
runner = "ubuntu-24.04-arm" # Free ARM64 runner for public repos (Public Preview)
elif arch == PPC64LE:
# GitHub doesn't provide PPC runners, would need self-hosted
runner = "self-hosted-ppc64le"
else: # AMD64
runner = "ubuntu-24.04" # Ubuntu 24.04 LTS
job = f""" test-linux{arch_suffix}:
runs-on: {runner}
strategy:
fail-fast: false
matrix:
include:
"""
for entry in matrix_entries:
job += " - env:\n"
for key, value in entry.items():
# Properly escape values with special characters
if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
job += f' {key}: "{value}"\n'
else:
job += f" {key}: {value}\n"
# Add manual job entries
manual_entries = [
{
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': '--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof',
'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
},
{
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': '--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof',
'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
},
{
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': 'force_tls=0',
'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
},
{
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': 'force_tls=0 --enable-debug',
'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
}
]
# --enable-cxx-infallible-new coverage. Plain variant runs on all arches;
# debug-combined variant runs only on AMD64 to bound matrix size.
infallible_new_entries = [
{
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
},
]
if arch == AMD64:
infallible_new_entries.append({
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': '--enable-cxx-infallible-new --enable-debug',
'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
})
if arch == AMD64:
for entry in manual_entries:
job += " - env:\n"
for key, value in entry.items():
if ' ' in str(value):
job += f' {key}: "{value}"\n'
else:
job += f" {key}: {value}\n"
for entry in infallible_new_entries:
job += " - env:\n"
for key, value in entry.items():
if ' ' in str(value):
job += f' {key}: "{value}"\n'
else:
job += f" {key}: {value}\n"
job += f"""
steps:
- uses: actions/checkout@v6
- name: Show OS version
run: |
echo "=== System Information ==="
uname -a
echo ""
echo "=== Architecture ==="
uname -m
arch
echo ""
echo "=== OS Release ==="
cat /etc/os-release || true
echo ""
echo "=== CPU Info ==="
lscpu | grep -E "Architecture|CPU op-mode|Byte Order|CPU\\(s\\):" || true
- name: Install dependencies (32-bit)
if: matrix.env.CROSS_COMPILE_32BIT == 'yes'
run: |
sudo dpkg --add-architecture i386
sudo apt-get update
sudo apt-get install -y gcc-multilib g++-multilib libc6-dev-i386
- name: Build and test
env:
CC: ${{{{ matrix.env.CC }}}}
CXX: ${{{{ matrix.env.CXX }}}}
COMPILER_FLAGS: ${{{{ matrix.env.COMPILER_FLAGS }}}}
CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}}
EXTRA_CFLAGS: ${{{{ matrix.env.EXTRA_CFLAGS }}}}
run: |
# Verify the script generates the same output
./scripts/gen_gh_actions.py > gh_actions_script.yml
# Run autoconf
autoconf
# Configure with flags
if [ -n "$COMPILER_FLAGS" ]; then
./configure CC="${{CC}} ${{COMPILER_FLAGS}}" CXX="${{CXX}} ${{COMPILER_FLAGS}}" $CONFIGURE_FLAGS
else
./configure $CONFIGURE_FLAGS
fi
# Build
make -j3
make -j3 tests
# Run tests
make check
"""
return job
def generate_macos_job(arch):
"""Generate macOS job configuration."""
os = OSX
max_unusual_opts = 1
exclude = ([Option.as_malloc_conf(opt) for opt in (
'dss:primary',
'background_thread:true')] +
[Option.as_configure_flag('--enable-prof')] +
[CLANG,])
if arch != ARM64:
exclude += [LARGE_HUGEPAGE]
matrix_entries = generate_job_matrix_entries(os, arch, exclude, max_unusual_opts)
arch_suffix = f"-{arch}" if arch != AMD64 else ""
# Select appropriate runner based on architecture
# Pin both for more control over OS upgrades
if arch == ARM64:
runner = "macos-15" # Pinned macOS 15 on Apple Silicon
else: # AMD64
runner = "macos-15-intel" # Pinned macOS 15 on Intel (last Intel runner, EOL Aug 2027)
job = f""" test-macos{arch_suffix}:
runs-on: {runner}
strategy:
fail-fast: false
matrix:
include:
"""
for entry in matrix_entries:
job += " - env:\n"
for key, value in entry.items():
if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
job += f' {key}: "{value}"\n'
else:
job += f" {key}: {value}\n"
# --enable-cxx-infallible-new coverage on macOS (both arches).
macos_extra_cflags = ' '.join(get_extra_cflags(OSX, GCC.value))
infallible_new_entries = [
{
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
'EXTRA_CFLAGS': macos_extra_cflags
},
]
for entry in infallible_new_entries:
job += " - env:\n"
for key, value in entry.items():
if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
job += f' {key}: "{value}"\n'
else:
job += f" {key}: {value}\n"
job += f"""
steps:
- uses: actions/checkout@v6
- name: Show OS version
run: |
echo "=== macOS Version ==="
sw_vers
echo ""
echo "=== Architecture ==="
uname -m
arch
echo ""
echo "=== CPU Info ==="
sysctl -n machdep.cpu.brand_string
sysctl -n hw.machine
- name: Install dependencies
run: |
brew install autoconf
- name: Build and test
env:
CC: ${{{{ matrix.env.CC || 'gcc' }}}}
CXX: ${{{{ matrix.env.CXX || 'g++' }}}}
COMPILER_FLAGS: ${{{{ matrix.env.COMPILER_FLAGS }}}}
CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}}
EXTRA_CFLAGS: ${{{{ matrix.env.EXTRA_CFLAGS }}}}
run: |
# Run autoconf
autoconf
# Configure with flags
if [ -n "$COMPILER_FLAGS" ]; then
./configure CC="${{CC}} ${{COMPILER_FLAGS}}" CXX="${{CXX}} ${{COMPILER_FLAGS}}" $CONFIGURE_FLAGS
else
./configure $CONFIGURE_FLAGS
fi
# Build
make -j3
make -j3 tests
# Run tests
make check
"""
return job
def generate_windows_job(arch):
"""Generate Windows job configuration."""
os = WINDOWS
max_unusual_opts = 3
unusuals = (
Option.as_configure_flag('--enable-debug'),
CL,
CROSS_COMPILE_32BIT,
)
matrix_entries = generate_job_matrix_entries(os, arch, (), max_unusual_opts, unusuals)
arch_suffix = f"-{arch}" if arch != AMD64 else ""
# Use latest for Windows - tends to be backward compatible and stable
job = f""" test-windows{arch_suffix}:
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include:
"""
for entry in matrix_entries:
job += " - env:\n"
for key, value in entry.items():
if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
job += f' {key}: "{value}"\n'
else:
job += f" {key}: {value}\n"
# --enable-cxx-infallible-new coverage on Windows (MinGW-GCC only).
windows_mingw_cflags = ' '.join(get_extra_cflags(WINDOWS, GCC.value))
infallible_new_entries = [
{
'CC': 'gcc',
'CXX': 'g++',
'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
'EXTRA_CFLAGS': windows_mingw_cflags
},
]
for entry in infallible_new_entries:
job += " - env:\n"
for key, value in entry.items():
if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
job += f' {key}: "{value}"\n'
else:
job += f" {key}: {value}\n"
job += f"""
steps:
- uses: actions/checkout@v6
- name: Show OS version
shell: cmd
run: |
echo === Windows Version ===
systeminfo | findstr /B /C:"OS Name" /C:"OS Version"
ver
echo.
echo === Architecture ===
echo PROCESSOR_ARCHITECTURE=%PROCESSOR_ARCHITECTURE%
echo.
- name: Setup MSYS2
uses: msys2/setup-msys2@v2
with:
msystem: ${{{{ matrix.env.CROSS_COMPILE_32BIT == 'yes' && 'MINGW32' || 'MINGW64' }}}}
update: true
install: >-
autotools
git
pacboy: >-
make:p
gcc:p
binutils:p
- name: Build and test (MinGW-GCC)
if: matrix.env.CC != 'cl.exe'
shell: msys2 {{0}}
env:
CC: ${{{{ matrix.env.CC || 'gcc' }}}}
CXX: ${{{{ matrix.env.CXX || 'g++' }}}}
COMPILER_FLAGS: ${{{{ matrix.env.COMPILER_FLAGS }}}}
CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}}
EXTRA_CFLAGS: ${{{{ matrix.env.EXTRA_CFLAGS }}}}
run: |
# Run autoconf
autoconf
# Configure with flags
if [ -n "$COMPILER_FLAGS" ]; then
./configure CC="${{CC}} ${{COMPILER_FLAGS}}" CXX="${{CXX}} ${{COMPILER_FLAGS}}" $CONFIGURE_FLAGS
else
./configure $CONFIGURE_FLAGS
fi
# Build (mingw32-make is the "make" command in MSYS2)
mingw32-make -j3
mingw32-make tests
# Run tests
mingw32-make -k check
- name: Setup MSVC environment
if: matrix.env.CC == 'cl.exe'
uses: ilammy/msvc-dev-cmd@v1
with:
arch: ${{{{ matrix.env.CROSS_COMPILE_32BIT == 'yes' && 'x86' || 'x64' }}}}
- name: Build and test (MSVC)
if: matrix.env.CC == 'cl.exe'
shell: msys2 {{0}}
env:
CONFIGURE_FLAGS: ${{{{ matrix.env.CONFIGURE_FLAGS }}}}
MSYS2_PATH_TYPE: inherit
run: |
# Export MSVC environment variables for configure
export CC=cl.exe
export CXX=cl.exe
export AR=lib.exe
export NM=dumpbin.exe
export RANLIB=:
# Verify cl.exe is accessible (should be in PATH via inherit)
if ! which cl.exe > /dev/null 2>&1; then
echo "cl.exe not found, trying to locate MSVC..."
# Find and add MSVC bin directory to PATH
MSVC_BIN=$(cmd.exe /c "echo %VCToolsInstallDir%" | tr -d '\\\\r' | sed 's/\\\\\\\\\\\\\\\\/\\//g' | sed 's/C:/\\\\/c/g')
if [ -n "$MSVC_BIN" ]; then
export PATH="$PATH:$MSVC_BIN/bin/Hostx64/x64:$MSVC_BIN/bin/Hostx86/x86"
fi
fi
# Run autoconf
autoconf
# Configure with MSVC
./configure CC=cl.exe CXX=cl.exe AR=lib.exe $CONFIGURE_FLAGS
# Build (mingw32-make is the "make" command in MSYS2)
mingw32-make -j3
# Build tests sequentially due to PDB file issues
mingw32-make tests
# Run tests
mingw32-make -k check
"""
return job
def generate_freebsd_job(arch):
"""Generate FreeBSD job configuration."""
# FreeBSD runs in a VM on ubuntu-latest, not native
job = f""" test-freebsd:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
debug: ['--enable-debug', '--disable-debug']
prof: ['--enable-prof', '--disable-prof']
arch: ['64-bit', '32-bit']
uncommon:
- ''
- '--with-lg-page=16 --with-malloc-conf=tcache:false'
name: FreeBSD (${{{{ matrix.arch }}}}, debug=${{{{ matrix.debug }}}}, prof=${{{{ matrix.prof }}}}${{{{ matrix.uncommon && ', uncommon' || '' }}}})
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 1
- name: Test on FreeBSD
uses: vmactions/freebsd-vm@v1
with:
release: '15.0'
usesh: true
prepare: |
pkg install -y autoconf gmake
run: |
# Verify we're running in FreeBSD
echo "==== System Information ===="
uname -a
freebsd-version
echo "============================"
# Set compiler flags for 32-bit if needed
if [ "${{{{ matrix.arch }}}}" = "32-bit" ]; then
export CC="cc -m32"
export CXX="c++ -m32"
fi
# Generate configure script
autoconf
# Configure with matrix options
./configure --with-jemalloc-prefix=ci_ ${{{{ matrix.debug }}}} ${{{{ matrix.prof }}}} ${{{{ matrix.uncommon }}}}
# Get CPU count for parallel builds
export JFLAG=$(sysctl -n kern.smp.cpus)
gmake -j${{JFLAG}}
gmake -j${{JFLAG}} tests
gmake check
"""
return job
def main():
import sys
# Determine which workflow to generate based on command-line argument
workflow_type = sys.argv[1] if len(sys.argv) > 1 else 'linux'
if workflow_type == 'linux':
jobs = '\n'.join((
generate_linux_job(AMD64),
generate_linux_job(ARM64),
))
print(GITHUB_ACTIONS_TEMPLATE.format(name='Linux CI', jobs=jobs))
elif workflow_type == 'macos':
jobs = '\n'.join((
generate_macos_job(AMD64), # Intel x86_64
generate_macos_job(ARM64), # Apple Silicon
))
print(GITHUB_ACTIONS_TEMPLATE.format(name='macOS CI', jobs=jobs))
elif workflow_type == 'windows':
jobs = generate_windows_job(AMD64)
print(GITHUB_ACTIONS_TEMPLATE.format(name='Windows CI', jobs=jobs))
elif workflow_type == 'freebsd':
jobs = generate_freebsd_job(AMD64)
print(GITHUB_ACTIONS_TEMPLATE.format(name='FreeBSD CI', jobs=jobs))
elif workflow_type == 'all':
# Generate all workflow files
linux_jobs = '\n'.join((
generate_linux_job(AMD64),
generate_linux_job(ARM64),
))
macos_jobs = '\n'.join((
generate_macos_job(AMD64), # Intel
generate_macos_job(ARM64), # Apple Silicon
))
windows_jobs = generate_windows_job(AMD64)
freebsd_jobs = generate_freebsd_job(AMD64)
all_jobs = '\n'.join((linux_jobs, macos_jobs, windows_jobs, freebsd_jobs))
print(GITHUB_ACTIONS_TEMPLATE.format(name='CI', jobs=all_jobs))
else:
print(f"Unknown workflow type: {workflow_type}", file=sys.stderr)
print("Usage: gen_gh_actions.py [linux|macos|windows|freebsd|all]", file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
main()