Merge pull request #34 from intel/prep210

changes for 2.10 release

.github/workflows/build-test.yml

@@ -10,9 +10,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
-      - name: Build builder image
-        run: |
-          builder/build_docker_image
       - name: Build svr-info
         run: |
           builder/build

.github/workflows/codeql.yml

@@ -23,25 +23,17 @@ jobs:
       security-events: write
     strategy:
       fail-fast: false
-      matrix:
-        language: [ 'cpp', 'go' ]
     steps:
     - name: Checkout repository
       uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12
       with:
-        languages: ${{ matrix.language }}
+        languages: go
     - name: Build golang
-      if: matrix.language == 'go'
       run: |
-        builder/build_docker_image
         builder/build
-    - name: Build c
-      if: matrix.language == 'cpp'
-      run: |
-        make calcfreq
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12
       with:

.gitignore

@@ -1,12 +1,12 @@
 dist
 bin
-build.*
 cmd/orchestrator/targets
 cmd/orchestrator/orchestrator_20*
 cmd/orchestrator/resources/collector_deps_*
 cmd/orchestrator/resources/collector
 cmd/orchestrator/resources/collector_arm64
 cmd/orchestrator/resources/sshpass
 cmd/orchestrator/resources/reporter
-cmd/pmu2metrics/scripts
 cmd/reporter/debug_out/*
+*.log
+targets

Makefile

@@ -12,13 +12,13 @@ VERSION := $(VERSION_NUMBER)_$(COMMIT_DATE)_$(COMMIT_ID)
 TARBALL := svr-info.tgz
 
 default: dist
-.PHONY: clean default dist dist-amd64 test tools
+.PHONY: clean default dist dist-amd64 test third_party
 
 bin:
 	mkdir -p bin
 
 orchestrator: bin reporter collector collector-deps
-	cp /prebuilt/bin/sshpass cmd/orchestrator/resources/
+	-cp /prebuilt/third-party/sshpass cmd/orchestrator/resources/
 	cp bin/reporter cmd/orchestrator/resources/
 	cp bin/collector cmd/orchestrator/resources/
 	cp bin/collector_arm64 cmd/orchestrator/resources/
@@ -35,35 +35,31 @@ msrread: bin
 	cd bin && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '-s -w -X main.gVersion=$(VERSION)' -o msrread ../cmd/msrread
 
 msrwrite: bin
-	cd bin && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '-s -w -X main.gVersion=$(VERSION)' -o msrwrite ../cmd/msrread
+	cd bin && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '-s -w -X main.gVersion=$(VERSION)' -o msrwrite ../cmd/msrwrite
 
 msrbusy: bin
 	cd bin && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '-s -w -X main.gVersion=$(VERSION)' -o msrbusy ../cmd/msrbusy
 
 pmu2metrics: bin
 	rm -f cmd/pmu2metrics/resources/perf
 	cd bin && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '-s -w -X main.gVersion=$(VERSION)' -o pmu2metrics_noperf ../cmd/pmu2metrics
-	-cp /prebuilt/bin/perf cmd/pmu2metrics/resources
+	-cp /prebuilt/third-party/perf cmd/pmu2metrics/resources
 	cd bin && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '-s -w -X main.gVersion=$(VERSION)' -o pmu2metrics ../cmd/pmu2metrics
 	rm -f cmd/pmu2metrics/resources/perf
 
-calcfreq: bin
-	cd bin && gcc -D VERSION=\"$(VERSION)\" ../tools/calcfreq/calcfreq.c -lpthread -o calcfreq -static
-
-collector-deps-amd64: third_party calcfreq msrbusy msrread msrwrite pmu2metrics
+collector-deps-amd64: msrbusy msrread msrwrite pmu2metrics
 	$(eval TMPDIR := $(shell mktemp -d build.XXXXXX))
-	cp -R /prebuilt/bin/* $(TMPDIR)
-	cp bin/calcfreq $(TMPDIR)
+	-cp -R /prebuilt/third-party/* $(TMPDIR)
 	cp bin/msrbusy $(TMPDIR)
 	cp bin/msrread $(TMPDIR)
 	cp bin/msrwrite $(TMPDIR)
 	cp bin/pmu2metrics_noperf $(TMPDIR)/pmu2metrics
 	cd $(TMPDIR) && tar -czf ../cmd/orchestrator/resources/collector_deps_amd64.tgz .
 	rm -rf $(TMPDIR)
 
-collector-deps-arm64: third_party
+collector-deps-arm64: 
 	$(eval TMPDIR := $(shell mktemp -d build.XXXXXX))
-	cp /prebuilt/bin/spectre-meltdown-checker.sh $(TMPDIR)
+	-cp /prebuilt/third-party/spectre-meltdown-checker.sh $(TMPDIR)
 	cd $(TMPDIR) && tar -czf ../cmd/orchestrator/resources/collector_deps_arm64.tgz .
 	rm -rf $(TMPDIR)
 
@@ -85,12 +81,12 @@ dist-amd64: orchestrator
 	cp bin/collector dist/svr-info/tools
 	cp bin/collector_arm64 dist/svr-info/tools
 	cp bin/reporter dist/svr-info/tools
-	cp bin/pmu2metrics dist/svr-info/tools
+	cp bin/pmu2metrics dist/svr-info/tools/pmu2metrics
 	cd dist && tar -czf $(TARBALL) svr-info
 	cd dist && md5sum $(TARBALL) > $(TARBALL).md5
 	rm -rf dist/svr-info
 
-dist: dist-amd64 third_party
+dist: dist-amd64
 	cp /prebuilt/oss_source.* dist
 
 clean:
@@ -101,7 +97,7 @@ test:
 	# test packages
 	cd internal/commandfile && go test -v -vet=all .
 	cd internal/core && go test -v -vet=all .
-	cd internal/cpu && go test -v -vet=all .
+	cd internal/cpudb && go test -v -vet=all .
 	# these tests require access to MSRs which we don't have on WSL2 and may not have on build machine 
 	# cd internal/msr && go test -v -vet=all .
 	cd internal/progress && go test -v -vet=all .

RELEASE_NOTES

@@ -2,10 +2,29 @@ RELEASE NOTES
 Intel® System Health Inspector (AKA svr-info)
 
 Fully Supported Platforms
-- Xeon Micro-Architectures: EMR,SPR,CPX,ICX,CLX,SKX,BDX,HSX
+- Xeon Micro-Architectures: SRF,EMR,SPR,CPX,ICX,CLX,SKX,BDX,HSX
 - Operating Systems: Ubuntu 16.04, 18.04, 20.04, 22.04, CentOS 7, Amazon Linux 2, Debian 11, RHEL 9, Rocky Linux 8
   Note: svr-info may work on other micro-architectures and Linux distributions, but has not been thoroughly tested
 
+2.10.0
+Features Added
+- Support for Sierra Forest Xeon CPUs
+- The '-benchmark frequency' option now measures AVX2/3 frequencies in addition to non-AVX frequencies
+- Minor improvements to System Summary field in Excel 'Brief' report
+- Nodes in multi-node HTML report will now be in the same order as presented in targets file
+- SRF and GNR ISAs identified in ISAs report table
+- EMR reference benchmark data added
+
+Other Changes
+- Removed "Feature" table from report
+- Removed Active Idle settings from report
+
+Known Issues
+- The '-profile pmu' and '-profile power' options do not function on SRF.
+- The storage micro-benchmark may not run on CentOS due to OS locale settings.
+- CPU cache sizes are reported in aggregate on Ubuntu 20.04 and newer.
+- DRAM population and CPU frequencies may not be accurate on some public cloud IaaS VMs.
+
 2.9.1
 Bugs Fixed
 - Extremely long run time on some EMR virtual machines

THIRD_PARTY_PROGRAMS

@@ -236,6 +236,38 @@ mpstat
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA              *
  ***************************************************************************
 -------------------------------------------------------------
+pcm
+BSD 3-Clause License
+
+Copyright (c) 2009-2024, Intel Corporation
+Copyright (c) 2016-2020, opcm
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+-------------------------------------------------------------
 perf
 The Linux Kernel is provided under:
 

builder/build

@@ -1,10 +1,23 @@
 #!/usr/bin/env bash
-docker container run                                  \
-    --volume "$(pwd)"/builder/scripts:/scripts        \
-    --volume "$(pwd)":/workdir                        \
-    --volume "/home/$USER/.ssh":/home/$USER/.ssh      \
-    --user $(id -u):$(id -g)                          \
+
+# run this script from repo's root directory
+# set GITHUB_ACCESS_TOKEN environment variable to your token, if you want to build MLC
+
+TAG=v1
+
+# build third-party components image
+docker build -f third_party/build.Dockerfile --build-arg GITHUB_ACCESS_TOKEN="${GITHUB_ACCESS_TOKEN}" --tag svr-info-third-party:$TAG ./third_party
+
+# build go cmd builder image
+docker build -f cmd/build.Dockerfile --tag svr-info-cmd-builder:$TAG .
+
+# build svr-info release package builder image
+docker build -f builder/build.Dockerfile --build-arg TAG=$TAG --tag svr-info-builder:$TAG .
+
+# build svr-info release package
+docker run                                            \
+    --volume "$(pwd)":/localrepo                      \
+    -w /localrepo                                     \
     --rm                                              \
-    --name build_svr_info4                            \
-    svr_info_builder:v4                               \
-    build
+    svr-info-builder:$TAG                             \
+    make dist test

builder/build.Dockerfile

@@ -0,0 +1,20 @@
+# image contains svr-info release package build environment
+# build image:
+#   $ docker build --build-arg TAG=v1 -f builder/build.Dockerfile --tag svr-info-builder:v1 .
+# build svr-info:
+#   $ docker run --rm -v "$PWD":/localrepo -w /localrepo svr-info-builder:v1 make dist
+
+ARG REGISTRY=
+ARG PREFIX=
+ARG TAG=
+# STAGE 1 - image contains pre-built third-party components, rebuild the image to rebuild the third-party components
+FROM ${REGISTRY}${PREFIX}svr-info-third-party:${TAG} AS third-party
+
+# STAGE 2- image contains svr-info's Go components build environment
+FROM ${REGISTRY}${PREFIX}svr-info-cmd-builder:${TAG} as svr-info
+RUN mkdir /prebuilt
+RUN mkdir /prebuilt/third-party
+RUN mkdir /prebuilt/bin
+COPY --from=third-party /bin/ /prebuilt/third-party
+COPY --from=third-party /oss_source* /prebuilt
+RUN git config --global --add safe.directory /localrepo

cmd/build.Dockerfile

@@ -0,0 +1,17 @@
+# image contains svr-info's Go components build environment, need to build them directly
+# build the image (from repo root directory): 
+#    $ docker image build -f cmd/build.Dockerfile --tag svr-info-cmd-builder:v1 .
+# build the svr-info Go components using this image
+#    $ docker run --rm -v "$PWD":/workdir -w /workdir svr-info-cmd-builder:v1 make dist
+
+FROM golang:1.22
+WORKDIR /workdir
+# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change
+COPY go.mod go.sum ./
+COPY internal/ ./internal/
+RUN go mod download && go mod verify
+# Radamsa is used for fuzz testing
+RUN curl -s https://gitlab.com/akihe/radamsa/uploads/a2228910d0d3c68d19c09cee3943d7e5/radamsa-0.6.c.gz | gzip -d | cc -O2 -x c -o /usr/local/bin/radamsa -
+# jq is needed in the functional test to inspect the svr-info json reports
+# zip is needed by CI/CD GHA
+RUN apt update && apt install -y jq zip

cmd/collector/collector_linux.go

@@ -31,7 +31,7 @@ func getUserPath() string {
 	return strings.Join(verifiedPaths, ":")
 }
 
-func runCommand(label string, command string, superuser bool, superuserPassword string, binPath string, timeout int) (stdout string, stderr string, exitCode int, err error) {
+func runCommand(command string, superuser bool, superuserPassword string, binPath string, timeout int) (stdout string, stderr string, exitCode int, err error) {
 	// explicitly set PATH by pre-pending to command
 	cmdWithPath := command
 	if binPath != "" {