Glue libprobe and probe_frontend together (#36)

* Glue libprobe and probe_frontend together * Fix nix flake checks * bundled-probe -> probe-bundled * Test empty commit * Fix stuff * Respond to comments * Fix nix flake checks * Fix Justfile * Fixed devshell for clean-slate installs * Fix stuff for direnv * Fix ruff check * Fix Mypy checks * Fix tests (paritally) * Fix devshell * Updated tests * Restore tests * Fix tests by starting at root node * Compile tests before running them
charmoniumQ · Aug 4, 2024 · 66e9713 · 66e9713
1 parent b35cf3f
commit 66e9713
Show file tree

Hide file tree

Showing 36 changed files with 528 additions and 1,139 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,3 +13,5 @@
 **/.directory
 **/.Trash*
 **/desktop.ini
+
+probe_log
diff --git a/Justfile b/Justfile
@@ -12,25 +12,43 @@ check-ruff:
     #ruff format --check probe_src # TODO: uncomment
     ruff check probe_src
 
+check-format-rust:
+    env --chdir probe_src/probe_frontend cargo fmt --check
+
+fix-format-rust:
+    env --chdir probe_src/probe_frontend cargo fmt
+
+check-clippy:
+    env --chdir probe_src/probe_frontend cargo clippy
+
+fix-clippy:
+    env --chdir probe_src/probe_frontend cargo clippy --fix --allow-staged
+
 check-mypy:
-    MYPYPATH=probe_src mypy --strict --package arena
-    MYPYPATH=probe_src mypy --strict --package probe_py
+    mypy --strict --package probe_py.manual
+    mypy --strict --package probe_py.generated
     mypy --strict probe_src/libprobe
 
-compile-libprobe:
+compile-lib:
     make --directory=probe_src/libprobe all
 
-test-ci: compile-libprobe
-    make --directory=probe_src/tests/c all
-    cd probe_src && python -m pytest .
+compile-cli:
+    env --chdir=probe_src/probe_frontend cargo build --release
 
-test-dev: compile-libprobe
+compile-tests:
     make --directory=probe_src/tests/c all
-    cd probe_src && python -m pytest . --failed-first --maxfail=1
+
+compile: compile-lib compile-cli compile-tests
+
+test-ci: compile-lib
+     pytest probe_src
+
+test-dev: compile-lib
+    pytest probe_src --failed-first --maxfail=1
 
 check-flake:
     nix flake check --all-systems
 
-pre-commit: fix-format-nix fix-ruff check-mypy check-flake compile-libprobe test-dev
+pre-commit: fix-format-nix   fix-ruff   fix-format-rust   fix-clippy   compile check-mypy             test-dev
 
-on-push: check-format-nix check-ruff check-mypy check-flake compile-libprobe test-ci
+on-push:    check-format-nix check-ruff check-format-rust check-clippy compile check-mypy check-flake test-ci
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -1,51 +1,127 @@
 {
-  inputs.flake-utils.url = "github:numtide/flake-utils";
+  inputs = {
+    nixpkgs = {
+      url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    };
+    flake-utils = {
+      url = "github:numtide/flake-utils";
+    };
+
+    crane = {
+      url = "github:ipetkov/crane";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
+    advisory-db = {
+      url = "github:rustsec/advisory-db";
+      flake = false;
+    };
+
+    rust-overlay = {
+      url = "github:oxalica/rust-overlay";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
 
   outputs = {
     self,
     nixpkgs,
     flake-utils,
-  }:
-    flake-utils.lib.eachDefaultSystem (
-      system: let
-        pkgs = nixpkgs.legacyPackages.${system};
-        inherit (pkgs) lib;
-        python312-debug = pkgs.python312.overrideAttrs (oldAttrs: {
-          configureFlags = oldAttrs.configureFlags ++ ["--with-pydebug"];
-          # patches = oldAttrs.patches ++ [ ./python.patch ];
-        });
-        export-and-rename = pkg: file-pairs:
-          pkgs.stdenv.mkDerivation {
-            pname = "${pkg.pname}-only-bin";
+    rust-overlay,
+    ...
+  }@inputs: let
+    supported-systems = [
+      "x86_64-linux"
+      "i686-linux"
+      "aarch64-linux"
+      "armv7l-linux"
+    ];
+  in
+    flake-utils.lib.eachSystem supported-systems (system: let
+      pkgs = import nixpkgs {
+        inherit system;
+        overlays = [(import rust-overlay)];
+      };
+      python = pkgs.python312;
+      frontend = (import ./probe_src/probe_frontend/frontend.nix) ({ inherit system pkgs python; } // inputs);
+    in {
+        packages = rec {
+          probe-bundled = let
+            # libprobe is a "private" package
+            # It is only used in probe-bundled
+            # TODO: The only public package should probably be probe-bundled and probe-py.
+            libprobe = pkgs.stdenv.mkDerivation rec {
+              pname = "libprobe";
+              version = "0.1.0";
+              src = ./probe_src/libprobe;
+              makeFlags = [ "INSTALL_PREFIX=$(out)" "SOURCE_VERSION=${version}" ];
+              buildInputs = [
+                (pkgs.python312.withPackages (pypkgs: [
+                  pypkgs.pycparser
+                ]))
+              ];
+            };
+          in pkgs.stdenv.mkDerivation rec {
+            pname = "probe-bundled";
+            version = "0.1.0";
             dontUnpack = true;
-            version = pkg.version;
-            buildInputs = [pkg];
-            buildPhase =
-              builtins.concatStringsSep
-              "\n"
-              (builtins.map
-                (pairs: "install -D ${pkg}/${builtins.elemAt pairs 0} $out/${builtins.elemAt pairs 1}")
-                file-pairs);
+            dontBuild = true;
+            nativeBuildInputs = [ pkgs.makeWrapper ];
+            installPhase = ''
+              mkdir $out $out/bin
+              makeWrapper \
+                ${self.packages.${system}.probe-cli}/bin/probe \
+                $out/bin/PROBE \
+                --set __PROBE_LIB ${libprobe}/lib
+            '';
           };
-      in {
-        packages = {
-          python-dbg = python312-debug;
-        };
+          probe-py-manual = python.pkgs.buildPythonPackage rec {
+            pname = "probe_py.manual";
+            version = "0.1.0";
+            pyproject = true;
+            build-system = [
+              python.pkgs.flit-core
+            ];
+            src = ./probe_src/python;
+            propagatedBuildInputs = [
+              self.packages.${system}.probe-py-generated
+              python.pkgs.networkx
+              python.pkgs.pygraphviz
+              python.pkgs.pydot
+              python.pkgs.rich
+              python.pkgs.typer
+            ];
+            pythonImportsCheck = [ pname ];
+          };
+          default = probe-bundled;
+        } // frontend.packages;
+        # TODO: Run pytest tests in Nix checks
+        checks = self.packages.${system} // frontend.checks;
         devShells = {
-          default = pkgs.mkShell {
+          default = frontend.devShells.default.overrideAttrs (oldAttrs: rec {
+            shellHook = ''
+              pushd $(git rev-parse --show-toplevel)
+              source ./setup_devshell.sh
+              popd
+            '';
             buildInputs =
-              [
-                (pkgs.python312.withPackages (pypkgs: [
-                  pypkgs.psutil
+              oldAttrs.buildInputs ++ [
+                (python.withPackages (pypkgs: [
+                  # probe_py.manual runtime requirements
+                  pypkgs.networkx
+                  pypkgs.pygraphviz
+                  pypkgs.pydot
+                  pypkgs.rich
                   pypkgs.typer
-                  pypkgs.pycparser
+
+                  # probe_py.manual "dev time" requirements
+                  pypkgs.psutil
                   pypkgs.pytest
                   pypkgs.mypy
-                  pypkgs.pygraphviz
-                  pypkgs.networkx
                   pypkgs.ipython
-                  pypkgs.pydot
-                  pypkgs.rich
+
+                  # libprobe build time requirement
+                  pypkgs.pycparser
                 ]))
                 # (export-and-rename python312-debug [["bin/python" "bin/python-dbg"]])
                 pkgs.which
@@ -67,12 +143,12 @@
               )
               ++ (
                 # while xdot isn't marked as linux only, it has a dependency (xvfb-run) that is
-                if builtins.elem system lib.platforms.linux
+                if builtins.elem system pkgs.lib.platforms.linux
                 then [pkgs.xdot]
                 else []
               );
-          };
+          });
         };
-      }
+       }
     );
 }
diff --git a/probe_src/PROBE b/probe_src/PROBE
diff --git a/probe_src/libprobe/Makefile b/probe_src/libprobe/Makefile
@@ -1,27 +1,29 @@
 SOURCE_VERSION ?= $(shell git rev-parse --short HEAD)
-CFLAGS ?= -DSOURCE_VERSION=\"$(SOURCE_VERSION)\" -Wno-cast-function-type -Wno-array-parameter -ffreestanding -Wl,--as-needed -Wall -Wextra -pthread
-DBGCFLAGS ?= $(CFLAGS) -Og -g
-OPTCFLAGS ?= $(CFLAGS) -O3 -DNDEBUG
-LIBCFLAGS ?= -fPIC -nostdlib -shared
-SOURCE_FILES := $(wildcard src/*.c) $(wildcard include/*.h)
+CFLAGS ?= -DSOURCE_VERSION=\"$(SOURCE_VERSION)\" -Wno-cast-function-type -Wno-array-parameter -ffreestanding -Wl,--as-needed -Wall -Wextra -pthread -fPIC -nostdlib -shared $(NIX_CFLAGS_COMPILE)
+DBGCFLAGS ?= -Og -g
+OPTCFLAGS ?= -O3 -DNDEBUG
 
+SOURCE_FILES := $(wildcard src/*.c) $(wildcard include/*.h)
 GENERATED_FILES := generated/libc_hooks.c generated/libc_hooks.h
 ALL_TARGETS := build/libprobe-dbg.so build/libprobe.so
 
 all: $(ALL_TARGETS)
 .PHONY: all
 
 build/lib%.so: $(SOURCE_FILES) $(GENERATED_FILES)
-	mkdir --parents build/
-	gcc $(OPTCFLAGS) $(LIBCFLAGS) -o $@ src/lib.c
+	mkdir --parents build
+	gcc $(CFLAGS) $(OPTCFLAGS) -o $@ src/lib.c
 
 build/lib%-dbg.so: $(SOURCE_FILES) $(GENERATED_FILES)
-	mkdir --parents build/
-	gcc $(DBGCFLAGS) $(LIBCFLAGS) -o $@ src/lib.c
+	mkdir --parents build
+	gcc $(CFLAGS) $(DBGCFLAGS) -o $@ src/lib.c
 
 $(GENERATED_FILES): $(wildcard generator/*)
 	./generator/gen_libc_hooks.py
 
+install:
+	install -D --target-directory $(INSTALL_PREFIX)/lib/ build/lib*.so
+
 clean:
 	mkdir --parents build/ generated/
 	touch $(GENERATED_FILES) $(ALL_TARGETS)

diff --git a/probe_src/arena/.gitignore → probe_src/libprobe/arena/.gitignore b/probe_src/arena/.gitignore → probe_src/libprobe/arena/.gitignore
diff --git a/probe_src/arena/Makefile → probe_src/libprobe/arena/Makefile b/probe_src/arena/Makefile → probe_src/libprobe/arena/Makefile
diff --git a/probe_src/arena/README.md → probe_src/libprobe/arena/README.md b/probe_src/arena/README.md → probe_src/libprobe/arena/README.md
diff --git a/probe_src/arena/arena.c → probe_src/libprobe/arena/include/arena.h b/probe_src/arena/arena.c → probe_src/libprobe/arena/include/arena.h
@@ -1,5 +1,4 @@
-#ifndef ARENA
-#define ARENA
+#pragma once
 
 #define _GNU_SOURCE
 #ifdef PYCPARSER
@@ -334,5 +333,3 @@ __attribute__((unused)) static bool arena_is_initialized(struct ArenaDir* arena_
     );
     return arena_dir->__tail != NULL;
 }
-
-#endif // ARENA
diff --git a/probe_src/arena/parse_arena.py → probe_src/libprobe/arena/parse_arena.py b/probe_src/arena/parse_arena.py → probe_src/libprobe/arena/parse_arena.py