From 93fa0158ebd36b91f925062d12e005b9f95c0106 Mon Sep 17 00:00:00 2001 From: TheK0tYaRa Date: Tue, 17 Mar 2026 19:26:06 +0200 Subject: [PATCH] Make electron clang base store-pure --- custom/override.nix | 265 +++++++++++++++++++++++++++++++++----------- todo.md | 33 ++++++ 2 files changed, 235 insertions(+), 63 deletions(-) create mode 100644 todo.md diff --git a/custom/override.nix b/custom/override.nix index ad13ce9..fc04874 100644 --- a/custom/override.nix +++ b/custom/override.nix @@ -20,6 +20,49 @@ let target.overrideScope f else throw "modify: target does not support overridePythonAttrs, overrideAttrs, or overrideScope"; + mkRealCompilerBasePath = + { + name, + toolchain, + }: + final.runCommand name { } '' + wrapped_clang="$(readlink -f ${toolchain}/bin/clang)" + wrapped_root="$(dirname "$(dirname "$wrapped_clang")")" + orig_cc_root="$(tr -d '\n' < "$wrapped_root/nix-support/orig-cc")" + + if [ -z "$orig_cc_root" ] || [ ! -d "$orig_cc_root/bin" ]; then + echo "mkRealCompilerBasePath: invalid orig-cc root: $orig_cc_root" >&2 + exit 1 + fi + if [ ! -x "$orig_cc_root/bin/clang" ] || [ ! -x "$orig_cc_root/bin/clang++" ]; then + echo "mkRealCompilerBasePath: clang or clang++ missing in $orig_cc_root/bin" >&2 + exit 1 + fi + + mkdir -p "$out/bin" + + for executable in ${toolchain}/bin/*; do + executable_name="$(basename "$executable")" + case "$executable_name" in + clang|clang++|cc|c++) + continue + ;; + esac + ln -s "$executable" "$out/bin/$executable_name" + done + + ln -s "$orig_cc_root/bin/clang" "$out/bin/clang" + ln -s "$orig_cc_root/bin/clang++" "$out/bin/clang++" + ln -s clang "$out/bin/cc" + ln -s clang++ "$out/bin/c++" + + for path in ${toolchain}/*; do + path_name="$(basename "$path")" + if [ "$path_name" != bin ]; then + ln -s "$path" "$out/$path_name" + fi + done + ''; mkSccacheLauncher = { name, @@ -161,75 +204,161 @@ let ${sccacheLauncher}/bin/cmake-sccache --stop-server || true ''; }; + mkSccacheExtraConfig = + { + sccacheDir ? sandboxSccacheDir, + extraConfig ? "", + }: + '' + mkdir -p ${final.lib.escapeShellArg sccacheDir} + chmod 0777 ${final.lib.escapeShellArg sccacheDir} || true + export SCCACHE_CONF=${final.lib.escapeShellArg ( + final.writeText "sccache-stdenv.conf" '' + [cache.disk] + dir = "${sccacheDir}" + size = "100G" + '' + )} + export SCCACHE_NO_DAEMON=1 + unset SCCACHE_SERVER_UDS + ${extraConfig} + ''; + mkWrappedCcForSccache = + { + cc, + extraConfig ? "", + }: + let + wrappedCompiler = final.sccache.links { + inherit extraConfig; + unwrappedCC = cc.cc; + }; + overrideArgs = cc.override.__functionArgs or { }; + in + if overrideArgs ? cc then + cc.override { cc = wrappedCompiler; } + else if overrideArgs ? llvm then + cc.override { llvm = wrappedCompiler; } + else + throw "mkWrappedCcForSccache: unsupported compiler wrapper override interface"; mkSccacheStdenv = { stdenv, sccacheDir ? "/var/cache/sccache/nix-builds/packages", - sccacheServerUds ? null, - noDaemon ? true, + extraConfig ? "", }: - let - cc = stdenv.cc; - ccName = "${cc.targetPrefix or ""}cc"; - cxxName = "${cc.targetPrefix or ""}c++"; - sccacheLauncher = mkSccacheLauncher { - name = "stdenv-sccache"; - inherit sccacheDir sccacheServerUds noDaemon; - }; - sccacheCc = cc.overrideAttrs (old: { - setupHooks = (old.setupHooks or [ ]) ++ [ - (final.writeText "sccache-wrapped-compiler-hook.sh" '' - set_sccache_passthrough_for_cmake_configure() { - case " ''${nativeBuildInputs:-} ''${propagatedNativeBuildInputs:-} " in - *"/cmake-"*) - export SCCACHE_WRAPPED_COMPILER_PASSTHROUGH=1 - ;; - *) - if [ -n "''${cmakeFlags:-}" ] || [ -n "''${cmakeDir:-}" ] || [ -n "''${cmakeBuildType:-}" ]; then - export SCCACHE_WRAPPED_COMPILER_PASSTHROUGH=1 - fi - ;; - esac - } - - clear_sccache_passthrough_after_configure() { - unset SCCACHE_WRAPPED_COMPILER_PASSTHROUGH - } - - preConfigureHooks+=(set_sccache_passthrough_for_cmake_configure) - postConfigureHooks+=(clear_sccache_passthrough_after_configure) - '') - ]; - nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ - final.makeWrapper - sccacheLauncher - ]; - postFixup = (old.postFixup or "") + '' - - wrap_with_sccache() { - local tool_path="$1" - if [ ! -x "$tool_path" ]; then - return 0 - fi - - mv "$tool_path" "$tool_path.orig" - makeWrapper ${sccacheLauncher}/bin/stdenv-sccache "$tool_path" \ - --add-flags "$tool_path.orig" - } - - wrap_with_sccache "$out/bin/${ccName}" - wrap_with_sccache "$out/bin/${cxxName}" - wrap_with_sccache "$out/bin/gcc" - wrap_with_sccache "$out/bin/g++" - wrap_with_sccache "$out/bin/clang" - wrap_with_sccache "$out/bin/clang++" - wrap_with_sccache "$out/bin/arocc" - ''; - }); - in - final.overrideCC stdenv sccacheCc; + final.overrideCC stdenv ( + mkWrappedCcForSccache { + inherit (stdenv) cc; + extraConfig = mkSccacheExtraConfig { + inherit extraConfig sccacheDir; + }; + } + ); in { + sccache = modify prev.sccache (old: { + postPatch = (old.postPatch or "") + '' + substituteInPlace src/compiler/gcc.rs \ + --replace-fail \ + ' take_arg!("-isystem", PathBuf, CanBeSeparated, PreprocessorArgumentPath),' \ + ' take_arg!("-isystem", PathBuf, CanBeSeparated, PreprocessorArgumentPath), + take_arg!("-cxx-isystem", PathBuf, CanBeSeparated, PreprocessorArgumentPath),' + ''; + passthru = (old.passthru or { }) // { + links = + { unwrappedCC, extraConfig ? "" }: + final.stdenv.mkDerivation { + pname = "sccache-links"; + inherit (old) version; + passthru = { + isClang = unwrappedCC.isClang or false; + isGNU = unwrappedCC.isGNU or false; + isSccache = true; + dev = unwrappedCC.dev or null; + lib = unwrappedCC.lib or (final.lib.getLib unwrappedCC); + } // final.lib.optionalAttrs (unwrappedCC ? rsrc) { + inherit (unwrappedCC) rsrc; + }; + dev = unwrappedCC.dev or null; + lib = unwrappedCC.lib or (final.lib.getLib unwrappedCC); + rsrc = unwrappedCC.rsrc or null; + nativeBuildInputs = [ final.makeWrapper ]; + buildCommand = + let + targetPrefix = + if unwrappedCC.isClang or false then + "" + else + (final.lib.optionalString ( + unwrappedCC ? targetConfig && unwrappedCC.targetConfig != null && unwrappedCC.targetConfig != "" + ) "${unwrappedCC.targetConfig}-"); + in + '' + mkdir -p $out/bin + + wrap() { + local cname="${targetPrefix}$1" + if [ -x "${unwrappedCC}/bin/$cname" ]; then + makeWrapper ${final.sccache}/bin/sccache $out/bin/$cname \ + --run ${final.lib.escapeShellArg extraConfig} \ + --add-flags ${unwrappedCC}/bin/$cname + fi + } + + wrap cc + wrap c++ + wrap gcc + wrap g++ + wrap clang + wrap clang++ + + for executable in $(ls ${unwrappedCC}/bin); do + if [ ! -x "$out/bin/$executable" ]; then + ln -s ${unwrappedCC}/bin/$executable $out/bin/$executable + fi + done + for file in $(ls ${unwrappedCC} | grep -vw bin); do + ln -s ${unwrappedCC}/$file $out/$file + done + ''; + meta = final.lib.optionalAttrs (unwrappedCC.meta ? mainProgram) { + inherit (unwrappedCC.meta) mainProgram; + }; + }; + }; + }); + sccacheWrapper = + final.lib.makeOverridable ( + { + extraConfig ? "", + cc, + }: + mkWrappedCcForSccache { + inherit cc extraConfig; + } + ) { + extraConfig = ""; + inherit (final.stdenv) cc; + }; + sccacheStdenv = final.lib.lowPrio ( + final.lib.makeOverridable ( + { + stdenv, + ... + }@extraArgs: + final.overrideCC stdenv ( + final.buildPackages.sccacheWrapper.override ( + { inherit (stdenv) cc; } + // final.lib.optionalAttrs (builtins.hasAttr "extraConfig" extraArgs) { + extraConfig = extraArgs.extraConfig; + } + ) + ) + ) { + inherit (final) stdenv; + } + ); sccache-config = sharedSccacheConfig; intel-sycl = modify prev."intel-sycl" ( syFinal: syPrev: { @@ -287,12 +416,22 @@ in electron_39 = let angleLib = final.lib.getLib final.angle; + realElectronClangBasePath = mkRealCompilerBasePath { + name = "electron-real-clang-base"; + toolchain = prev.electron_39.passthru.unwrapped.llvmCcAndBintools; + }; electronUnwrapped = modify prev.electron_39.passthru.unwrapped ( old: let buildSccacheAttrs = mkBuildSccacheAttrs old; + oldClangBasePath = "clang_base_path=\"${old.llvmCcAndBintools}\""; + newClangBasePath = "clang_base_path=\"${realElectronClangBasePath}\""; + baseGnFlags = builtins.replaceStrings + [ oldClangBasePath ] + [ newClangBasePath ] + (old.gnFlags or ""); electronGnFlags = - "${old.gnFlags or ""} cc_wrapper=\"${buildSccacheLauncher}/bin/build-sccache\""; + "${baseGnFlags} cc_wrapper=\"${buildSccacheLauncher}/bin/build-sccache\""; in buildSccacheAttrs // { diff --git a/todo.md b/todo.md new file mode 100644 index 0000000..5a42fc0 --- /dev/null +++ b/todo.md @@ -0,0 +1,33 @@ +# Todo + +## Sccache Stdenv Direction + +- Goal: move `sccache` to the same structural layer as `ccacheStdenv`, meaning between the Nix cc-wrapper and the real compiler, instead of wrapping the cc-wrapper script itself. +- Reason: this should preserve normal compiler identity and wrapper semantics while still allowing cached compilations. +- Constraint: prefer direct local-disk cache access for sandboxed builds and avoid depending on the external `sccache` client/server socket model. + +## Current Implementation State + +- Added a local patch to `sccache` so its GCC/Clang parser treats `-cxx-isystem ` like `-isystem `. +- Added `sccache.links`, modeled after nixpkgs `ccache.links`, to generate `cc`/`c++`/`gcc`/`g++`/`clang`/`clang++` wrappers that call `sccache` with the real compiler path. +- Added `sccacheWrapper`, modeled after nixpkgs `ccacheWrapper`. +- Added `sccacheStdenv`, modeled after nixpkgs `ccacheStdenv`. +- Reworked `mkSccacheStdenv` to delegate to `sccacheWrapper` instead of patching cc-wrapper scripts directly. +- Kept direct-disk cache configuration through generated `SCCACHE_CONF` and `SCCACHE_NO_DAEMON=1`. +- Adjusted the wrapper override helper so it can handle both normal cc-wrappers (`cc = ...`) and Intel SYCL wrappers (`llvm = ...`). +- Forwarded Intel-expected outputs/attrs like `.dev`, `.lib`, and `.rsrc` through `sccache.links`. + +## Verified Findings + +- The previous direct insertion into cc-wrapper final exec did not break hello-world compilation. +- That previous attempt was not cache-effective because `sccache` interpreted wrapper-emitted `-cxx-isystem` arguments as extra input files and returned `multiple input files`. +- The exact problematic arguments were confirmed with a direct `sccache clang ...` probe using the wrapper-emitted argv. +- The new ccache-style `sccacheWrapper` / `sccacheStdenv` path evaluates successfully. +- `intel-sycl.stdenv.cc` now also evaluates successfully with the ccache-style `sccache.links` replacement in place. + +## Next Checks + +- `electron-real-clang-base` should stay store-pure: synthesize its output tree from symlinks only and never `cp`/`rm` against copied toolchain store content. +- Build the patched `sccache` package. +- Run a minimal hello-world derivation with the new `sccacheStdenv` and confirm repeated compiles produce cache hits. +- If that works, switch current consumers to the ccache-style path and remove obsolete wrapper-patching logic.