Make electron clang base store-pure
This commit is contained in:
parent
6e63bb0a90
commit
93fa0158eb
2 changed files with 235 additions and 63 deletions
|
|
@ -20,6 +20,49 @@ let
|
|||
target.overrideScope f
|
||||
else
|
||||
throw "modify: target does not support overridePythonAttrs, overrideAttrs, or overrideScope";
|
||||
mkRealCompilerBasePath =
|
||||
{
|
||||
name,
|
||||
toolchain,
|
||||
}:
|
||||
final.runCommand name { } ''
|
||||
wrapped_clang="$(readlink -f ${toolchain}/bin/clang)"
|
||||
wrapped_root="$(dirname "$(dirname "$wrapped_clang")")"
|
||||
orig_cc_root="$(tr -d '\n' < "$wrapped_root/nix-support/orig-cc")"
|
||||
|
||||
if [ -z "$orig_cc_root" ] || [ ! -d "$orig_cc_root/bin" ]; then
|
||||
echo "mkRealCompilerBasePath: invalid orig-cc root: $orig_cc_root" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -x "$orig_cc_root/bin/clang" ] || [ ! -x "$orig_cc_root/bin/clang++" ]; then
|
||||
echo "mkRealCompilerBasePath: clang or clang++ missing in $orig_cc_root/bin" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$out/bin"
|
||||
|
||||
for executable in ${toolchain}/bin/*; do
|
||||
executable_name="$(basename "$executable")"
|
||||
case "$executable_name" in
|
||||
clang|clang++|cc|c++)
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
ln -s "$executable" "$out/bin/$executable_name"
|
||||
done
|
||||
|
||||
ln -s "$orig_cc_root/bin/clang" "$out/bin/clang"
|
||||
ln -s "$orig_cc_root/bin/clang++" "$out/bin/clang++"
|
||||
ln -s clang "$out/bin/cc"
|
||||
ln -s clang++ "$out/bin/c++"
|
||||
|
||||
for path in ${toolchain}/*; do
|
||||
path_name="$(basename "$path")"
|
||||
if [ "$path_name" != bin ]; then
|
||||
ln -s "$path" "$out/$path_name"
|
||||
fi
|
||||
done
|
||||
'';
|
||||
mkSccacheLauncher =
|
||||
{
|
||||
name,
|
||||
|
|
@ -161,75 +204,161 @@ let
|
|||
${sccacheLauncher}/bin/cmake-sccache --stop-server || true
|
||||
'';
|
||||
};
|
||||
mkSccacheExtraConfig =
|
||||
{
|
||||
sccacheDir ? sandboxSccacheDir,
|
||||
extraConfig ? "",
|
||||
}:
|
||||
''
|
||||
mkdir -p ${final.lib.escapeShellArg sccacheDir}
|
||||
chmod 0777 ${final.lib.escapeShellArg sccacheDir} || true
|
||||
export SCCACHE_CONF=${final.lib.escapeShellArg (
|
||||
final.writeText "sccache-stdenv.conf" ''
|
||||
[cache.disk]
|
||||
dir = "${sccacheDir}"
|
||||
size = "100G"
|
||||
''
|
||||
)}
|
||||
export SCCACHE_NO_DAEMON=1
|
||||
unset SCCACHE_SERVER_UDS
|
||||
${extraConfig}
|
||||
'';
|
||||
mkWrappedCcForSccache =
|
||||
{
|
||||
cc,
|
||||
extraConfig ? "",
|
||||
}:
|
||||
let
|
||||
wrappedCompiler = final.sccache.links {
|
||||
inherit extraConfig;
|
||||
unwrappedCC = cc.cc;
|
||||
};
|
||||
overrideArgs = cc.override.__functionArgs or { };
|
||||
in
|
||||
if overrideArgs ? cc then
|
||||
cc.override { cc = wrappedCompiler; }
|
||||
else if overrideArgs ? llvm then
|
||||
cc.override { llvm = wrappedCompiler; }
|
||||
else
|
||||
throw "mkWrappedCcForSccache: unsupported compiler wrapper override interface";
|
||||
mkSccacheStdenv =
|
||||
{
|
||||
stdenv,
|
||||
sccacheDir ? "/var/cache/sccache/nix-builds/packages",
|
||||
sccacheServerUds ? null,
|
||||
noDaemon ? true,
|
||||
extraConfig ? "",
|
||||
}:
|
||||
let
|
||||
cc = stdenv.cc;
|
||||
ccName = "${cc.targetPrefix or ""}cc";
|
||||
cxxName = "${cc.targetPrefix or ""}c++";
|
||||
sccacheLauncher = mkSccacheLauncher {
|
||||
name = "stdenv-sccache";
|
||||
inherit sccacheDir sccacheServerUds noDaemon;
|
||||
final.overrideCC stdenv (
|
||||
mkWrappedCcForSccache {
|
||||
inherit (stdenv) cc;
|
||||
extraConfig = mkSccacheExtraConfig {
|
||||
inherit extraConfig sccacheDir;
|
||||
};
|
||||
sccacheCc = cc.overrideAttrs (old: {
|
||||
setupHooks = (old.setupHooks or [ ]) ++ [
|
||||
(final.writeText "sccache-wrapped-compiler-hook.sh" ''
|
||||
set_sccache_passthrough_for_cmake_configure() {
|
||||
case " ''${nativeBuildInputs:-} ''${propagatedNativeBuildInputs:-} " in
|
||||
*"/cmake-"*)
|
||||
export SCCACHE_WRAPPED_COMPILER_PASSTHROUGH=1
|
||||
;;
|
||||
*)
|
||||
if [ -n "''${cmakeFlags:-}" ] || [ -n "''${cmakeDir:-}" ] || [ -n "''${cmakeBuildType:-}" ]; then
|
||||
export SCCACHE_WRAPPED_COMPILER_PASSTHROUGH=1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
clear_sccache_passthrough_after_configure() {
|
||||
unset SCCACHE_WRAPPED_COMPILER_PASSTHROUGH
|
||||
}
|
||||
|
||||
preConfigureHooks+=(set_sccache_passthrough_for_cmake_configure)
|
||||
postConfigureHooks+=(clear_sccache_passthrough_after_configure)
|
||||
'')
|
||||
];
|
||||
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
|
||||
final.makeWrapper
|
||||
sccacheLauncher
|
||||
];
|
||||
postFixup = (old.postFixup or "") + ''
|
||||
|
||||
wrap_with_sccache() {
|
||||
local tool_path="$1"
|
||||
if [ ! -x "$tool_path" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
mv "$tool_path" "$tool_path.orig"
|
||||
makeWrapper ${sccacheLauncher}/bin/stdenv-sccache "$tool_path" \
|
||||
--add-flags "$tool_path.orig"
|
||||
}
|
||||
|
||||
wrap_with_sccache "$out/bin/${ccName}"
|
||||
wrap_with_sccache "$out/bin/${cxxName}"
|
||||
wrap_with_sccache "$out/bin/gcc"
|
||||
wrap_with_sccache "$out/bin/g++"
|
||||
wrap_with_sccache "$out/bin/clang"
|
||||
wrap_with_sccache "$out/bin/clang++"
|
||||
wrap_with_sccache "$out/bin/arocc"
|
||||
'';
|
||||
});
|
||||
in
|
||||
final.overrideCC stdenv sccacheCc;
|
||||
);
|
||||
in
|
||||
{
|
||||
sccache = modify prev.sccache (old: {
|
||||
postPatch = (old.postPatch or "") + ''
|
||||
substituteInPlace src/compiler/gcc.rs \
|
||||
--replace-fail \
|
||||
' take_arg!("-isystem", PathBuf, CanBeSeparated, PreprocessorArgumentPath),' \
|
||||
' take_arg!("-isystem", PathBuf, CanBeSeparated, PreprocessorArgumentPath),
|
||||
take_arg!("-cxx-isystem", PathBuf, CanBeSeparated, PreprocessorArgumentPath),'
|
||||
'';
|
||||
passthru = (old.passthru or { }) // {
|
||||
links =
|
||||
{ unwrappedCC, extraConfig ? "" }:
|
||||
final.stdenv.mkDerivation {
|
||||
pname = "sccache-links";
|
||||
inherit (old) version;
|
||||
passthru = {
|
||||
isClang = unwrappedCC.isClang or false;
|
||||
isGNU = unwrappedCC.isGNU or false;
|
||||
isSccache = true;
|
||||
dev = unwrappedCC.dev or null;
|
||||
lib = unwrappedCC.lib or (final.lib.getLib unwrappedCC);
|
||||
} // final.lib.optionalAttrs (unwrappedCC ? rsrc) {
|
||||
inherit (unwrappedCC) rsrc;
|
||||
};
|
||||
dev = unwrappedCC.dev or null;
|
||||
lib = unwrappedCC.lib or (final.lib.getLib unwrappedCC);
|
||||
rsrc = unwrappedCC.rsrc or null;
|
||||
nativeBuildInputs = [ final.makeWrapper ];
|
||||
buildCommand =
|
||||
let
|
||||
targetPrefix =
|
||||
if unwrappedCC.isClang or false then
|
||||
""
|
||||
else
|
||||
(final.lib.optionalString (
|
||||
unwrappedCC ? targetConfig && unwrappedCC.targetConfig != null && unwrappedCC.targetConfig != ""
|
||||
) "${unwrappedCC.targetConfig}-");
|
||||
in
|
||||
''
|
||||
mkdir -p $out/bin
|
||||
|
||||
wrap() {
|
||||
local cname="${targetPrefix}$1"
|
||||
if [ -x "${unwrappedCC}/bin/$cname" ]; then
|
||||
makeWrapper ${final.sccache}/bin/sccache $out/bin/$cname \
|
||||
--run ${final.lib.escapeShellArg extraConfig} \
|
||||
--add-flags ${unwrappedCC}/bin/$cname
|
||||
fi
|
||||
}
|
||||
|
||||
wrap cc
|
||||
wrap c++
|
||||
wrap gcc
|
||||
wrap g++
|
||||
wrap clang
|
||||
wrap clang++
|
||||
|
||||
for executable in $(ls ${unwrappedCC}/bin); do
|
||||
if [ ! -x "$out/bin/$executable" ]; then
|
||||
ln -s ${unwrappedCC}/bin/$executable $out/bin/$executable
|
||||
fi
|
||||
done
|
||||
for file in $(ls ${unwrappedCC} | grep -vw bin); do
|
||||
ln -s ${unwrappedCC}/$file $out/$file
|
||||
done
|
||||
'';
|
||||
meta = final.lib.optionalAttrs (unwrappedCC.meta ? mainProgram) {
|
||||
inherit (unwrappedCC.meta) mainProgram;
|
||||
};
|
||||
};
|
||||
};
|
||||
});
|
||||
sccacheWrapper =
|
||||
final.lib.makeOverridable (
|
||||
{
|
||||
extraConfig ? "",
|
||||
cc,
|
||||
}:
|
||||
mkWrappedCcForSccache {
|
||||
inherit cc extraConfig;
|
||||
}
|
||||
) {
|
||||
extraConfig = "";
|
||||
inherit (final.stdenv) cc;
|
||||
};
|
||||
sccacheStdenv = final.lib.lowPrio (
|
||||
final.lib.makeOverridable (
|
||||
{
|
||||
stdenv,
|
||||
...
|
||||
}@extraArgs:
|
||||
final.overrideCC stdenv (
|
||||
final.buildPackages.sccacheWrapper.override (
|
||||
{ inherit (stdenv) cc; }
|
||||
// final.lib.optionalAttrs (builtins.hasAttr "extraConfig" extraArgs) {
|
||||
extraConfig = extraArgs.extraConfig;
|
||||
}
|
||||
)
|
||||
)
|
||||
) {
|
||||
inherit (final) stdenv;
|
||||
}
|
||||
);
|
||||
sccache-config = sharedSccacheConfig;
|
||||
intel-sycl = modify prev."intel-sycl" (
|
||||
syFinal: syPrev: {
|
||||
|
|
@ -287,12 +416,22 @@ in
|
|||
electron_39 =
|
||||
let
|
||||
angleLib = final.lib.getLib final.angle;
|
||||
realElectronClangBasePath = mkRealCompilerBasePath {
|
||||
name = "electron-real-clang-base";
|
||||
toolchain = prev.electron_39.passthru.unwrapped.llvmCcAndBintools;
|
||||
};
|
||||
electronUnwrapped = modify prev.electron_39.passthru.unwrapped (
|
||||
old:
|
||||
let
|
||||
buildSccacheAttrs = mkBuildSccacheAttrs old;
|
||||
oldClangBasePath = "clang_base_path=\"${old.llvmCcAndBintools}\"";
|
||||
newClangBasePath = "clang_base_path=\"${realElectronClangBasePath}\"";
|
||||
baseGnFlags = builtins.replaceStrings
|
||||
[ oldClangBasePath ]
|
||||
[ newClangBasePath ]
|
||||
(old.gnFlags or "");
|
||||
electronGnFlags =
|
||||
"${old.gnFlags or ""} cc_wrapper=\"${buildSccacheLauncher}/bin/build-sccache\"";
|
||||
"${baseGnFlags} cc_wrapper=\"${buildSccacheLauncher}/bin/build-sccache\"";
|
||||
in
|
||||
buildSccacheAttrs
|
||||
// {
|
||||
|
|
|
|||
33
todo.md
Normal file
33
todo.md
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# Todo
|
||||
|
||||
## Sccache Stdenv Direction
|
||||
|
||||
- Goal: move `sccache` to the same structural layer as `ccacheStdenv`, meaning between the Nix cc-wrapper and the real compiler, instead of wrapping the cc-wrapper script itself.
|
||||
- Reason: this should preserve normal compiler identity and wrapper semantics while still allowing cached compilations.
|
||||
- Constraint: prefer direct local-disk cache access for sandboxed builds and avoid depending on the external `sccache` client/server socket model.
|
||||
|
||||
## Current Implementation State
|
||||
|
||||
- Added a local patch to `sccache` so its GCC/Clang parser treats `-cxx-isystem <dir>` like `-isystem <dir>`.
|
||||
- Added `sccache.links`, modeled after nixpkgs `ccache.links`, to generate `cc`/`c++`/`gcc`/`g++`/`clang`/`clang++` wrappers that call `sccache` with the real compiler path.
|
||||
- Added `sccacheWrapper`, modeled after nixpkgs `ccacheWrapper`.
|
||||
- Added `sccacheStdenv`, modeled after nixpkgs `ccacheStdenv`.
|
||||
- Reworked `mkSccacheStdenv` to delegate to `sccacheWrapper` instead of patching cc-wrapper scripts directly.
|
||||
- Kept direct-disk cache configuration through generated `SCCACHE_CONF` and `SCCACHE_NO_DAEMON=1`.
|
||||
- Adjusted the wrapper override helper so it can handle both normal cc-wrappers (`cc = ...`) and Intel SYCL wrappers (`llvm = ...`).
|
||||
- Forwarded Intel-expected outputs/attrs like `.dev`, `.lib`, and `.rsrc` through `sccache.links`.
|
||||
|
||||
## Verified Findings
|
||||
|
||||
- The previous direct insertion into cc-wrapper final exec did not break hello-world compilation.
|
||||
- That previous attempt was not cache-effective because `sccache` interpreted wrapper-emitted `-cxx-isystem` arguments as extra input files and returned `multiple input files`.
|
||||
- The exact problematic arguments were confirmed with a direct `sccache clang ...` probe using the wrapper-emitted argv.
|
||||
- The new ccache-style `sccacheWrapper` / `sccacheStdenv` path evaluates successfully.
|
||||
- `intel-sycl.stdenv.cc` now also evaluates successfully with the ccache-style `sccache.links` replacement in place.
|
||||
|
||||
## Next Checks
|
||||
|
||||
- `electron-real-clang-base` should stay store-pure: synthesize its output tree from symlinks only and never `cp`/`rm` against copied toolchain store content.
|
||||
- Build the patched `sccache` package.
|
||||
- Run a minimal hello-world derivation with the new `sccacheStdenv` and confirm repeated compiles produce cache hits.
|
||||
- If that works, switch current consumers to the ccache-style path and remove obsolete wrapper-patching logic.
|
||||
Loading…
Add table
Add a link
Reference in a new issue