Skip to content

Commit

Permalink
Introduce the LRE flake overlay
Browse files Browse the repository at this point in the history
This change migrates the last patch to an overlay and removes the patch
structure. This resolves all IFDs, cleans up the main flake and makes
our custom toolchains portable across flakes.

The `customClang` and `customStdenv` packages are now available as
`lre.clang` and `lre.stdenv`.

While the new packages are differently named, they produce the exact
same setup as previously. This can be verified by the fact that
`generate-toolchains` is a noop between the previous commit and this
one.

Since a renaming of the `customClang` executable itself would cause
differences in the lre-cc toolchain we defer this to a future commit.

Fixes TraceMachina#1262
Closes TraceMachina#1512
  • Loading branch information
aaronmondal committed Dec 3, 2024
1 parent 301e51b commit 99f93cb
Show file tree
Hide file tree
Showing 7 changed files with 141 additions and 115 deletions.
44 changes: 16 additions & 28 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,6 @@

llvmPackages = pkgs.llvmPackages_19;

customStdenv = pkgs.callPackage ./tools/llvmStdenv.nix {inherit llvmPackages;};

# TODO(aaronmondal): This doesn't work with rules_rust yet.
# Tracked in https://github.com/TraceMachina/nativelink/issues/477.
customClang = pkgs.callPackage ./tools/customClang.nix {stdenv = customStdenv;};

nixSystemToRustTriple = nixSystem:
{
"x86_64-linux" = "x86_64-unknown-linux-musl";
Expand Down Expand Up @@ -251,7 +245,7 @@

rbe-autogen = pkgs.callPackage ./local-remote-execution/rbe-autogen.nix {
inherit buildImage;
stdenv = customStdenv;
inherit (pkgs.lre) stdenv;
};
createWorker = pkgs.callPackage ./tools/create-worker.nix {inherit buildImage self;};
buck2-toolchain = let
Expand Down Expand Up @@ -292,8 +286,7 @@
};
};
lre-cc = pkgs.callPackage ./local-remote-execution/lre-cc.nix {
inherit customClang buildImage;
stdenv = customStdenv;
inherit buildImage;
};
toolchain-drake = buildImage {
name = "toolchain-drake";
Expand Down Expand Up @@ -348,23 +341,15 @@

nativelinkCoverageForHost = nativelinkCoverageFor pkgs;
in rec {
_module.args.pkgs = let
nixpkgs-patched = (import self.inputs.nixpkgs {inherit system;}).applyPatches {
name = "nixpkgs-patched";
src = self.inputs.nixpkgs;
patches = [
./tools/nixpkgs_link_libunwind_and_libcxx.diff
];
};
in
import nixpkgs-patched {
inherit system;
overlays = [
(import ./tools/nixpkgs-disable-ratehammering-pulumi-tests.nix)
(import rust-overlay)
(import ./tools/rust-overlay-cut-libsecret.nix)
];
};
_module.args.pkgs = import self.inputs.nixpkgs {
inherit system;
overlays = [
self.overlays.lre
(import ./tools/nixpkgs-disable-ratehammering-pulumi-tests.nix)
(import rust-overlay)
(import ./tools/rust-overlay-cut-libsecret.nix)
];
};
apps = {
default = {
type = "app";
Expand Down Expand Up @@ -446,7 +431,7 @@
"/run/current-system/sw/bin"
"${binutils.bintools}/bin"
"${uutils-coreutils-noprefix}/bin"
"${customClang}/bin"
"${pkgs.lre.clang}/bin"
"${git}/bin"
"${python3}/bin"
];
Expand Down Expand Up @@ -497,7 +482,7 @@
# Additional tools from within our development environment.
local-image-test
generate-toolchains
customClang
pkgs.lre.clang
native-cli
docs
build-chromium-tests
Expand Down Expand Up @@ -560,5 +545,8 @@
local-remote-execution = ./local-remote-execution/flake-module.nix;
nixos = ./tools/nixos/flake-module.nix;
};
overlays = {
lre = import ./local-remote-execution/overlays/default.nix;
};
};
}
11 changes: 5 additions & 6 deletions local-remote-execution/lre-cc.nix
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
{
buildImage,
customClang,
stdenv,
lre,
lib,
coreutils,
findutils,
Expand All @@ -15,15 +14,15 @@
# binary identical toolchains during local and remote execution.
("PATH="
+ (lib.strings.concatStringsSep ":" [
"${stdenv.cc.bintools}/bin"
"${customClang}/bin"
"${stdenv}/bin"
"${lre.stdenv.cc.bintools}/bin"
"${lre.clang}/bin"
"${lre.stdenv}/bin"
"${coreutils}/bin"
"${findutils}/bin"
"${gnutar}/bin"
]))

"CC=${customClang}/bin/customClang"
"CC=${lre.clang}/bin/customClang"
];
in
buildImage {
Expand Down
File renamed without changes.
12 changes: 12 additions & 0 deletions local-remote-execution/overlays/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
final: _prev: {
lre = {
stdenv = final.callPackage ./stdenv.nix {
llvmPackages = final.llvmPackages_19;
targetPackages = final;
};

clang = final.callPackage ./clang.nix {
inherit (final.lre) stdenv;
};
};
}
108 changes: 108 additions & 0 deletions local-remote-execution/overlays/stdenv.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
{
lib,
llvmPackages,
overrideCC,
stdenv,
targetPackages,
useMoldLinker,
wrapCCWith,
}: let
# Adapted from clangUseLLVM.
# See: https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/compilers/llvm/common/default.nix
clangVersion = "19";
targetLlvmLibraries = targetPackages.llvmPackages_19.libraries;

mkExtraBuildCommands0 = cc: ''
rsrc="$out/resource-root"
mkdir "$rsrc"
ln -s "${lib.getLib cc}/lib/clang/${clangVersion}/include" "$rsrc"
echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags
'';
mkExtraBuildCommands = cc:
mkExtraBuildCommands0 cc
+ ''
ln -s "${targetLlvmLibraries.compiler-rt.out}/lib" "$rsrc/lib"
ln -s "${targetLlvmLibraries.compiler-rt.out}/share" "$rsrc/share"
'';

toolchain' = wrapCCWith (
rec {
cc = llvmPackages.clang-unwrapped;
inherit (targetLlvmLibraries) libcxx;
inherit (llvmPackages) bintools;
extraPackages = [
targetLlvmLibraries.compiler-rt
targetLlvmLibraries.libunwind
];
extraBuildCommands = mkExtraBuildCommands cc;
}
// {
nixSupport.cc-cflags =
[
"-rtlib=compiler-rt"
"-Wno-unused-command-line-argument"
"-B${targetLlvmLibraries.compiler-rt}/lib"
"-stdlib=libc++"
]
++ lib.optional stdenv.targetPlatform.isLinux "-fuse-ld=mold"
++ lib.optional (
!stdenv.targetPlatform.isWasm && !stdenv.targetPlatform.isFreeBSD
) "--unwindlib=libunwind"
++ lib.optional (
!stdenv.targetPlatform.isWasm
&& !stdenv.targetPlatform.isFreeBSD
&& stdenv.targetPlatform.useLLVM or false
) ["-lunwind" "-lc++"]
++ lib.optional stdenv.targetPlatform.isWasm "-fno-exceptions";
nixSupport.cc-ldflags =
lib.optionals (
!stdenv.targetPlatform.isWasm && !stdenv.targetPlatform.isFreeBSD
) [
"-L${targetLlvmLibraries.libunwind}/lib"
"-rpath"
"${targetLlvmLibraries.libunwind}/lib"
"-L${targetLlvmLibraries.libcxx}/lib"
"-rpath"
"${targetLlvmLibraries.libcxx}/lib"
];
}
);

stdenv' =
overrideCC
llvmPackages.libcxxStdenv
toolchain';

toolchain =
if stdenv.targetPlatform.isDarwin
then stdenv' # Mold doesn't support darwin.
else useMoldLinker stdenv';
in
# This toolchain uses Clang as compiler, Mold as linker, libc++ as C++
# standard library and compiler-rt as compiler runtime. Resulting rust
# binaries depend dynamically linked on the nixpkgs distribution of glibc.
# C++ binaries additionally depend dynamically on libc++, libunwind and
# libcompiler-rt. Due to a bug we also depend on libgcc_s.
#
# TODO(aaronmondal): At the moment this toolchain is only used for the Cargo
# build. The Bazel build uses a different mostly hermetic LLVM toolchain. We
# should merge the two by generating the Bazel cc_toolchain from this stdenv.
# This likely requires a rewrite of
# https://github.com/bazelbuild/bazel-toolchains as the current implementation
# has poor compatibility with custom container images and doesn't support
# generating toolchain configs from image archives.
#
# TODO(aaronmondal): Due to various issues in the nixpkgs LLVM toolchains
# we're not getting a pure Clang/LLVM toolchain here. My guess is that the
# runtimes were not built with the degenerate LLVM toolchain but with the
# regular GCC stdenv from nixpkgs.
#
# For instance, outputs depend on libgcc_s since libcxx seems to have been was
# built with a GCC toolchain. We're also not using builtin atomics, or at
# least we're redundantly linking libatomic.
#
# Fix this as it fixes a large number of issues, including better
# cross-platform compatibility, reduced closure size, and
# static-linking-friendly licensing. This requires building the llvm project
# with the correct multistage bootstrapping process.
toolchain
49 changes: 0 additions & 49 deletions tools/llvmStdenv.nix

This file was deleted.

32 changes: 0 additions & 32 deletions tools/nixpkgs_link_libunwind_and_libcxx.diff

This file was deleted.

0 comments on commit 99f93cb

Please sign in to comment.