Allow running sbomnix without realising out paths

Allow running sbomnix, nixgraph, vulnxscan, nix_outdated, and
nix_secupdates without realising the target output if `--buildtime`
was requested.

This change also adds a fix to allow allow installing sbomnix tooling in
nix development shell in editable mode.

Signed-off-by: Henri Rosten <henri.rosten@unikie.com>

nixgraph/main.py

@@ -8,10 +8,8 @@
 
 import argparse
 import pathlib
-import sys
 from nixgraph.graph import NixDependencies
 from sbomnix.utils import (
-    LOG,
     set_log_verbosity,
     get_py_pkg_version,
     check_positive,
@@ -83,11 +81,9 @@ def main():
     """main entry point"""
     args = getargs()
     set_log_verbosity(args.verbose)
-    if not args.NIX_PATH.exists():
-        LOG.fatal("Invalid path: '%s'", args.NIX_PATH)
-        sys.exit(1)
     target_path = args.NIX_PATH.resolve().as_posix()
-    exit_unless_nix_artifact(target_path)
+    runtime = args.buildtime is False
+    exit_unless_nix_artifact(target_path, force_realise=runtime)
     deps = NixDependencies(target_path, args.buildtime)
     deps.graph(args)
 

sbomnix/main.py

@@ -8,7 +8,6 @@
 
 import argparse
 import pathlib
-import sys
 from sbomnix.sbomdb import SbomDb
 from sbomnix.utils import (
     LOG,
@@ -81,19 +80,15 @@ def main():
     """main entry point"""
     args = getargs()
     set_log_verbosity(args.verbose)
-    if not args.NIX_PATH.exists():
-        LOG.fatal("Invalid path: '%s'", args.NIX_PATH)
-        sys.exit(1)
     target_path = args.NIX_PATH.resolve().as_posix()
-    exit_unless_nix_artifact(target_path)
+    runtime = args.type in ("runtime", "both")
+    buildtime = args.type in ("buildtime", "both")
+    exit_unless_nix_artifact(target_path, force_realise=runtime)
     if not args.meta:
         LOG.warning(
             "Command line argument '--meta' missing: SBOM will not include "
             "license information (see '--help' for more details)"
         )
-    runtime = args.type in ("runtime", "both")
-    buildtime = args.type in ("buildtime", "both")
-
     sbomdb = SbomDb(target_path, runtime, buildtime, args.meta, args.depth)
     if args.cdx:
         sbomdb.to_cdx(args.cdx)

sbomnix/nix.py

@@ -97,11 +97,6 @@ def find_deriver(path):
     LOG.debug(path)
     if path.endswith(".drv"):
         return path
-    # Deriver from QueryPathInfo
-    qpi_deriver = exec_cmd(["nix-store", "-qd", path]).strip()
-    LOG.debug("qpi_deriver: %s", qpi_deriver)
-    if qpi_deriver and qpi_deriver != "unknown-deriver" and os.path.exists(qpi_deriver):
-        return qpi_deriver
     # Deriver from QueryValidDerivers
     ret = exec_cmd(["nix", "show-derivation", path], raise_on_error=False)
     if not ret:
@@ -115,6 +110,11 @@ def find_deriver(path):
     LOG.debug("qvd_deriver: %s", qvd_deriver)
     if qvd_deriver and os.path.exists(qvd_deriver):
         return qvd_deriver
+    # Deriver from QueryPathInfo
+    qpi_deriver = exec_cmd(["nix-store", "-qd", path]).strip()
+    LOG.debug("qpi_deriver: %s", qpi_deriver)
+    if qpi_deriver and qpi_deriver != "unknown-deriver" and os.path.exists(qpi_deriver):
+        return qpi_deriver
 
     error = ""
     if qpi_deriver and qpi_deriver != "unknown-deriver":

sbomnix/utils.py

@@ -126,9 +126,17 @@ def exec_cmd(cmd, raise_on_error=True, return_error=False):
         return None
 
 
-def exit_unless_nix_artifact(path):
-    """Exit with error if `path` is not a nix artifact"""
-    cmd = ["nix-store", "-q", path]
+def exit_unless_nix_artifact(path, force_realise=False):
+    """
+    Exit with error if `path` is not a nix artifact. If `force_realize` is True,
+    run the nix-store-query command with `--force-realize` realising the `path`
+    argument before running query.
+    """
+    LOG.debug("force_realize: %s", force_realise)
+    if force_realise:
+        cmd = ["nix-store", "-qf", path]
+    else:
+        cmd = ["nix-store", "-q", path]
     try:
         exec_cmd(cmd)
         return

scripts/env.sh

@@ -35,7 +35,10 @@ else
 fi
 # Remove duplicates from the PYTHONPATH, preserving order
 PYTHONPATH=$(remove_dups "$PYTHONPATH")
-echo "export PYTHONPATH=$PYTHONPATH"
 export PYTHONPATH="$PYTHONPATH"
+# Add all subdirs of REPOROOTDIR/scripts/ to PATH
+for d in "$REPOROOTDIR/scripts/"*; do if [ -d "$d" ]; then PATH="$d:$PATH"; fi; done
+PATH=$(remove_dups "$PATH")
+export PATH="$PATH"
 
 ################################################################################

scripts/nixupdate/nix_outdated.py

@@ -62,10 +62,7 @@ def getargs():
         "that would have an update in the package's upstream repository."
     )
     parser.add_argument("--local", help=helps, action="store_true")
-    helps = (
-        "Include target's buildtime dependencies to the scan. "
-        "By default, only runtime dependencies are considered."
-    )
+    helps = "Scan target buildtime instead of runtime dependencies."
     parser.add_argument("--buildtime", help=helps, action="store_true")
     helps = "Path to output file (default: ./nix_outdated.csv)"
     parser.add_argument("--out", nargs="?", help=helps, default="nix_outdated.csv")
@@ -77,9 +74,8 @@ def getargs():
 ################################################################################
 
 
-def _generate_sbom(target_path, buildtime=False):
+def _generate_sbom(target_path, runtime=True, buildtime=False):
     LOG.info("Generating SBOM for target '%s'", target_path)
-    runtime = True
     sbomdb = SbomDb(target_path, runtime, buildtime, meta_path=None)
     prefix = "nixdeps_"
     suffix = ".cdx.json"
@@ -254,9 +250,12 @@ def main():
     args = getargs()
     set_log_verbosity(args.verbose)
     target_path_abs = args.NIXPATH.resolve().as_posix()
-    exit_unless_nix_artifact(target_path_abs)
+    runtime = args.buildtime is False
+    dtype = "runtime" if runtime else "buildtime"
+    LOG.info("Checking %s dependencies referenced by '%s'", dtype, target_path_abs)
+    exit_unless_nix_artifact(target_path_abs, force_realise=runtime)
 
-    sbom_path = _generate_sbom(target_path_abs, args.buildtime)
+    sbom_path = _generate_sbom(target_path_abs, runtime, args.buildtime)
     LOG.info("Using SBOM '%s'", sbom_path)
 
     repology_out_path = _run_repology_cli(sbom_path)

scripts/nixupdate/nix_secupdates.py

@@ -61,10 +61,7 @@ def getargs():
         "not enabled by default."
     )
     parser.add_argument("--pr", help=helps, action="store_true")
-    helps = (
-        "Include target's buildtime dependencies to the scan. "
-        "By default, only runtime dependencies are considered."
-    )
+    helps = "Scan target buildtime instead of runtime dependencies."
     parser.add_argument("--buildtime", help=helps, action="store_true")
     helps = "Path to output file (default: ./nix_secupdates.csv)"
     parser.add_argument("--out", nargs="?", help=helps, default="nix_secupdates.csv")
@@ -400,7 +397,11 @@ def main():
     """main entry point"""
     args = getargs()
     set_log_verbosity(args.verbose)
-    exit_unless_nix_artifact(args.NIXPATH.resolve().as_posix())
+    runtime = args.buildtime is False
+    nix_path = args.NIXPATH.resolve().as_posix()
+    dtype = "runtime" if runtime else "buildtime"
+    LOG.info("Checking %s dependencies referenced by '%s'", dtype, nix_path)
+    exit_unless_nix_artifact(nix_path, runtime)
     df = _find_secupdates(args)
     _report(df)
     df_to_csv_file(df, args.out)

scripts/vulnxscan/vulnxscan.py

@@ -50,8 +50,8 @@ def getargs():
     helps = "Path to output file (default: ./vulns.csv)"
     parser.add_argument("--out", nargs="?", help=helps, default="vulns.csv")
     helps = (
-        "Include target's buildtime dependencies to the scan. "
-        "By default, only runtime dependencies are scanned."
+        "Scan target buildtime instead of runtime dependencies. This option "
+        "has no impact if the scan target is SBOM (ref: --sbom)."
     )
     parser.add_argument("--buildtime", help=helps, action="store_true")
     helps = (
@@ -156,7 +156,9 @@ def _parse_osv(self, df_osv):
             self.df_osv["scanner"] = "osv"
             self.df_osv.replace(np.nan, "", regex=True, inplace=True)
             self.df_osv.drop_duplicates(keep="first", inplace=True)
-            self.df_osv["modified"] = pd.to_datetime(self.df_osv["modified"])
+            self.df_osv["modified"] = pd.to_datetime(
+                self.df_osv["modified"], format="ISO8601"
+            )
             LOG.log(LOG_SPAM, "osv data:\n%s", self.df_osv.to_markdown())
             LOG.debug("OSV scan found vulnerabilities")
             if LOG.level <= logging.DEBUG:
@@ -306,9 +308,8 @@ def _is_patched(row):
     return False
 
 
-def _generate_sbom(target_path, buildtime=False):
+def _generate_sbom(target_path, runtime=True, buildtime=False):
     LOG.info("Generating SBOM for target '%s'", target_path)
-    runtime = True
     sbomdb = SbomDb(target_path, runtime, buildtime, meta_path=None)
     prefix = "vulnxscan_"
     cdx_suffix = ".json"
@@ -367,8 +368,11 @@ def main():
         sbom_cdx_path = target_path_abs
         sbom_csv_path = None
     else:
-        exit_unless_nix_artifact(target_path_abs)
-        sbom_cdx_path, sbom_csv_path = _generate_sbom(target_path_abs, args.buildtime)
+        runtime = args.buildtime is False
+        exit_unless_nix_artifact(target_path_abs, force_realise=runtime)
+        sbom_cdx_path, sbom_csv_path = _generate_sbom(
+            target_path_abs, runtime, args.buildtime
+        )
         LOG.info("Using cdx SBOM '%s'", sbom_cdx_path)
         LOG.info("Using csv SBOM '%s'", sbom_csv_path)
         scanner.scan_vulnix(target_path_abs, args.buildtime)