Merge pull request #18 from intel/prep271

Prepare for 2.7.1 release

RELEASE_NOTES

@@ -7,6 +7,22 @@ Fully Supported Platforms
 - Operating Systems: Ubuntu 16.04, 18.04, 20.04, 22.04, CentOS 7, Amazon Linux 2, Debian 11, RHEL 9, Rocky Linux 8
   Note: svr-info may work on other micro-architectures and Linux distributions, but has not been thoroughly tested
 
+2.7.1
+Features Added
+- None
+Bugs Fixed
+- CXL devices were not being detected due to lspci version incompatibility
+- Insights that rely on CPU micro-architecture names were not matching appended MCC/XCC text
+- Align power-perf policy insight to labels introduced in 2.7.0
+Minor Changes
+- Use latest (as of release build time) spectre-meltdown-checker script from public repo
+- IAX accelerator renamed to IAA
+- Fix information text regarding Perl on HTML Analysis report
+Known Issues
+- The storage micro-benchmark may not run on CentOS due to OS locale settings.
+- CPU cache sizes are reported in aggregate on Ubuntu 20.04 and newer.
+- DRAM population and CPU frequencies may not be accurate on some public cloud IaaS VMs.
+
 2.7.0
 Features Added
 - Power & Performance Setting Labels refined. Now includes "Performance", "Balance Performance", "Normal", "Normal Powersave", "Balance Powersave", and "Powersave". Report output also includes specific EPB value, e.g., "Performance (6)".

builder/Dockerfile

@@ -26,7 +26,7 @@ RUN cp /usr/local/lib/libz.a /usr/lib/x86_64-linux-gnu/libz.a
 RUN curl -s https://gitlab.com/akihe/radamsa/uploads/a2228910d0d3c68d19c09cee3943d7e5/radamsa-0.6.c.gz | gzip -d | cc -O2 -x c -o /usr/local/bin/radamsa -
 
 # Install Go
-ARG GO_VERSION="1.20.7"
+ARG GO_VERSION="1.21.0"
 RUN wget https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz
 RUN tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz
 RUN rm go${GO_VERSION}.linux-amd64.tar.gz

go.work.sum

@@ -9,5 +9,7 @@ golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU
 golang.org/x/image v0.0.0-20220902085622-e7cb96979f69/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
 golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
 golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=

src/Makefile

@@ -155,6 +155,8 @@ else
 	cd lspci && git checkout master && git pull
 endif
 	cd lspci && make
+	cd lspci && ./update-pciids.sh
+	cd lspci && gzip -c pci.ids > pci.ids.gz
 
 mlc:
 ifeq ("$(wildcard mlc)","")
@@ -173,11 +175,11 @@ perf: linux-source
 	cd linux/tools/perf && make LDFLAGS=-static
 
 spectre-meltdown-checker:
-	mkdir -p spectre-meltdown-checker
-	rm -f spectre-meltdown-checker/spectre-meltdown-checker.sh
-	# get script from this PR https://github.com/speed47/spectre-meltdown-checker/pull/418
-	cd spectre-meltdown-checker && wget https://raw.githubusercontent.com/speed47/spectre-meltdown-checker/3c4f5e4d8e0fc6fc828c5bc164f20372eb2537ac/spectre-meltdown-checker.sh
-	chmod +x spectre-meltdown-checker/spectre-meltdown-checker.sh
+ifeq ("$(wildcard spectre-meltdown-checker)","")
+	git clone https://github.com/speed47/spectre-meltdown-checker.git
+else
+	cd spectre-meltdown-checker && git checkout master && git pull
+endif
 
 sshpass:
 ifeq ("$(wildcard sshpass)","")
@@ -230,6 +232,7 @@ collector-deps-amd64: collector_tools
 	cp ipmitool/src/ipmitool.static $(TMPDIR)/ipmitool
 	cp lshw/src/lshw-static $(TMPDIR)/lshw
 	cp lspci/lspci $(TMPDIR)
+	cp lspci/pci.ids.gz $(TMPDIR)
 	-cp mlc/mlc $(TMPDIR)
 	cp msrbusy/msrbusy $(TMPDIR)
 	cp linux/tools/perf/perf $(TMPDIR)

src/orchestrator/collection.go

@@ -62,6 +62,11 @@ func customizeCommandYAML(cmdTemplate []byte, cmdLineArgs *CmdLineArgs, targetBi
 	cf.Args.Timeout = cmdLineArgs.cmdTimeout
 	for idx := range cf.Commands {
 		cmd := &cf.Commands[idx]
+		// set path to the lspci data file
+		if cmd.Label == "lspci -vmm" {
+			cmd.Command = fmt.Sprintf("lspci -i %s -vmm", filepath.Join(targetBinDir, "pci.ids.gz"))
+		}
+		// optional collection
 		if cmd.Label == "Memory MLC Bandwidth" || cmd.Label == "Memory MLC Loaded Latency Test" {
 			cmd.Run = strings.Contains(cmdLineArgs.benchmark, "memory") || strings.Contains(cmdLineArgs.benchmark, "all")
 		} else if cmd.Label == "stress-ng cpu methods" {

src/orchestrator/go.mod

@@ -16,7 +16,7 @@ replace intel.com/svr-info/pkg/commandfile => ../pkg/commandfile
 
 require (
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
-	golang.org/x/term v0.10.0
+	golang.org/x/term v0.11.0
 	gopkg.in/yaml.v2 v2.4.0
 	intel.com/svr-info/pkg/commandfile v0.0.0-00010101000000-000000000000
 	intel.com/svr-info/pkg/core v0.0.0-00010101000000-000000000000
@@ -27,6 +27,6 @@ require (
 require (
 	github.com/creasty/defaults v1.6.0 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 )

src/orchestrator/go.sum

@@ -1,14 +1,18 @@
 github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BSc=
 github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

src/orchestrator/resources/collector_reports.yaml.tmpl

@@ -276,7 +276,7 @@ commands:
   - label: lspci devices
     command: lspci -d 8086:3258 | wc -l
     parallel: true 
-  - label: iax devices
+  - label: iaa devices
     command: ls -1 /dev/iax
     parallel: true
   - label: dsa devices

src/pkg/progress/go.mod

@@ -2,6 +2,6 @@ module intel.com/svr-info/pkg/progress/v2
 
 go 1.20
 
-require golang.org/x/term v0.10.0
+require golang.org/x/term v0.11.0
 
-require golang.org/x/sys v0.10.0 // indirect
+require golang.org/x/sys v0.11.0 // indirect

src/pkg/progress/go.sum

@@ -1,4 +1,4 @@
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=

src/reporter/report_generator_html.go

@@ -34,7 +34,6 @@ const (
 )
 
 const noDataFound = "No data found."
-const perlWarning = "<br>Check if perl is installed in /usr/bin/."
 
 type ReportGeneratorHTML struct {
 	reports   []*Report
@@ -1020,7 +1019,7 @@ func (n *Node) MarshalJSON() ([]byte, error) {
 	})
 }
 
-func convertFoldedToJson(folded string) (out string, err error) {
+func convertFoldedToJSON(folded string) (out string, err error) {
 	rootNode := Node{Name: "root", Value: 0, Children: make(map[string]*Node)}
 	scanner := bufio.NewScanner(strings.NewReader(folded))
 	for scanner.Scan() {
@@ -1051,12 +1050,9 @@ func renderFlameGraph(header string, hv *HostValues, field string, hostIndex int
 	folded := hv.Values[0][fieldIdx]
 	if folded == "" {
 		out += noDataFound
-		if header == "System" {
-			out += perlWarning
-		}
 		return
 	}
-	jsonStacks, err := convertFoldedToJson(folded)
+	jsonStacks, err := convertFoldedToJSON(folded)
 	if err != nil {
 		log.Printf("failed to convert folded data: %v", err)
 		out += "Error."

src/reporter/report_tables.go

@@ -814,7 +814,7 @@ func newISATable(sources []*Source, category TableCategory) (table *Table) {
 	type ISA struct {
 		Name     string
 		FullName string
-		CpuID    string
+		CPUID    string
 		lscpu    string
 	}
 	isas := []ISA{
@@ -841,7 +841,7 @@ func newISATable(sources []*Source, category TableCategory) (table *Table) {
 		}
 		flags := source.valFromRegexSubmatch("lscpu", `^Flags.*:\s*(.*)$`)
 		for _, isa := range isas {
-			cpuSupport := yesIfTrue(source.valFromRegexSubmatch("cpuid -1", isa.CpuID+`\s*= (.+?)$`))
+			cpuSupport := yesIfTrue(source.valFromRegexSubmatch("cpuid -1", isa.CPUID+`\s*= (.+?)$`))
 			kernelSupport := "Yes"
 			match, err := regexp.MatchString(" "+isa.lscpu+" ", flags)
 			if err != nil {
@@ -1586,10 +1586,17 @@ func newVulnerabilityTable(sources []*Source, category TableCategory) (table *Ta
 			ValueNames: []string{},
 			Values:     [][]string{},
 		}
+		vulns := source.getVulnerabilities()
 		var values []string
-		for _, pair := range source.valsArrayFromRegexSubmatch("spectre-meltdown-checker", `(CVE-\d+-\d+): (.+)`) {
-			hostValues.ValueNames = append(hostValues.ValueNames, pair[0])
-			values = append(values, pair[1])
+		// sort the keys
+		var keys []string
+		for k := range vulns {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, k := range keys {
+			hostValues.ValueNames = append(hostValues.ValueNames, k)
+			values = append(values, vulns[k])
 		}
 		if len(values) > 0 {
 			hostValues.Values = append(hostValues.Values, []string{})

src/reporter/report_tables_helpers.go

@@ -653,7 +653,7 @@ func getMicroArchitecture(cpusInfo *cpu.CPU, family, model, stepping, capid4, de
 
 func getMicroArchitectureExt(family, model, sockets string, capid4 string, devices string) (uarch string, err error) {
 	if family != "6" || (model != "143" && model != "207" && model != "173") {
-		err = fmt.Errorf("No extended architecture info for %s:%s", family, model)
+		err = fmt.Errorf("no extended architecture info for %s:%s", family, model)
 		return
 	}
 	var capid4Int, bits int64

src/reporter/resources/accelerators.yaml

@@ -15,7 +15,7 @@
   full_name: Intel Data Streaming Accelerator
   description:  a high-performance data copy and transformation accelerator
 
-- name: IAX
+- name: IAA
   mfgid: 8086
   devid: CFE
   full_name: Intel Analytics Accelerator

src/reporter/resources/insights.grl

@@ -77,7 +77,7 @@ rule Vulnerabilities {
 rule PowerPerfPolicy {
 	when
 		Report.GetValue("Configuration", "Power", "Power & Perf Policy") != "" &&
-		Report.GetValue("Configuration", "Power", "Power & Perf Policy") != "Performance"
+		!Report.GetValue("Configuration", "Power", "Power & Perf Policy").Contains("Performance")
 	then
 		Report.AddInsight(
 			"Power and Performance policy is set to '" + Report.GetValue("Configuration", "Power", "Power & Perf Policy") + "'.",
@@ -156,16 +156,16 @@ rule MountDiscard {
 		Retract("MountDiscard");
 }
 
-rule IAXEnabled {
+rule IAAEnabled {
 	when
-		Report.GetValueFromColumnAsInt("Configuration", "Accelerator", "Name", "IAX", "Count") != 0 &&
-		Report.GetValueFromColumn("Configuration", "Accelerator", "Name", "IAX", "Work Queues") == "None"
+		Report.GetValueFromColumnAsInt("Configuration", "Accelerator", "Name", "IAA", "Count") != 0 &&
+		Report.GetValueFromColumn("Configuration", "Accelerator", "Name", "IAA", "Work Queues") == "None"
 	then
 		Report.AddInsight(
-			"No work queues are configured for IAX accelerator(s).",
-			"Consider configuring IAX to allow accelerated compression and decompression in IAX-enabled software."
+			"No work queues are configured for IAA accelerator(s).",
+			"Consider configuring IAA to allow accelerated compression and decompression in IAA-enabled software."
 		);
-		Retract("IAXEnabled");
+		Retract("IAAEnabled");
 }
 
 rule DSAEnabled {

src/reporter/resources/report.html.tmpl

@@ -237,6 +237,7 @@
     <div id="analyze" class="tabcontent tabnotdefault">
         <main class="content">
             <h3>Use the "-analyze all" option to collect all analysis data. See "-help" for finer control.</h3>
+            <h3>Note: Perl is required on the target machine to collapse the call stacks used to produce System Flame Graphs.</h3>
             Upload your Intel&reg; System Health Inspector JSON-formatted reports to <a href=https://optimizations.intel.com/gProfilerAnalyzer target="_blank" rel="noopener noreferrer">Intel&reg; Optimization Hub</a> for deeper analysis.
             {{$reportGen := .}}
             {{range .AnalyzeReport.Tables}}

src/reporter/rules_engine_context.go

@@ -234,10 +234,13 @@ func (r *RulesEngineContext) CompareMicroarchitecture(x, y string) int {
 	}
 	var xArch, yArch int
 	var ok bool
-	if xArch, ok = uArchs[x]; !ok {
+	if len(x) < 3 || len(y) < 3 {
 		return -2
 	}
-	if yArch, ok = uArchs[y]; !ok {
+	if xArch, ok = uArchs[x[0:3]]; !ok {
+		return -2
+	}
+	if yArch, ok = uArchs[y[0:3]]; !ok {
 		return -2
 	}
 	if xArch < yArch {

src/reporter/source.go

@@ -523,12 +523,12 @@ func (s *Source) getL3LscpuMB() (val float64, err error) {
 	re := regexp.MustCompile(`(\d+\.?\d*)\s*(\w+).*`) // match known formats
 	match := re.FindStringSubmatch(l3Lscpu)
 	if len(match) == 0 {
-		err = fmt.Errorf("Unknown L3 format in lscpu: %s", l3Lscpu)
+		err = fmt.Errorf("unknown L3 format in lscpu: %s", l3Lscpu)
 		return
 	}
 	l3SizeNoUnit, err := strconv.ParseFloat(match[1], 64)
 	if err != nil {
-		err = fmt.Errorf("Failed to parse L3 size from lscpu: %s, %v", l3Lscpu, err)
+		err = fmt.Errorf("failed to parse L3 size from lscpu: %s, %v", l3Lscpu, err)
 		return
 	}
 	if strings.ToLower(match[2][:1]) == "m" {
@@ -539,7 +539,7 @@ func (s *Source) getL3LscpuMB() (val float64, err error) {
 		val = l3SizeNoUnit / 1024
 		return
 	}
-	err = fmt.Errorf("Unknown L3 units in lscpu: %s", l3Lscpu)
+	err = fmt.Errorf("unknown L3 units in lscpu: %s", l3Lscpu)
 	return
 }
 
@@ -552,12 +552,12 @@ func (s *Source) getL3MSRMB(uArch string) (val float64, err error) {
 	l3MSRHex := s.getCommandOutputLine("rdmsr 0xc90")
 	l3MSR, err := strconv.ParseInt(l3MSRHex, 16, 64)
 	if err != nil {
-		err = fmt.Errorf("Failed to parse MSR output: %s", l3MSRHex)
+		err = fmt.Errorf("failed to parse MSR output: %s", l3MSRHex)
 		return
 	}
 	cacheWays := s.getCacheWays(uArch)
 	if len(cacheWays) == 0 {
-		err = fmt.Errorf("Failed to get cache ways for uArch: %s", uArch)
+		err = fmt.Errorf("failed to get cache ways for uArch: %s", uArch)
 		return
 	}
 	cpul3SizeGB := l3LscpuMB / 1024
@@ -568,7 +568,7 @@ func (s *Source) getL3MSRMB(uArch string) (val float64, err error) {
 			return
 		}
 	}
-	err = fmt.Errorf("Did not find %d in cache ways.", l3MSR)
+	err = fmt.Errorf("did not find %d in cache ways", l3MSR)
 	return
 }
 
@@ -604,7 +604,7 @@ func (s *Source) getL3PerCore(uArch string, coresPerSocketStr string, socketsStr
 		return
 	}
 	cacheMB := l3 / float64(coresPerSocket*sockets)
-	val = fmt.Sprintf("%s", strconv.FormatFloat(cacheMB, 'f', 3, 64))
+	val = strconv.FormatFloat(cacheMB, 'f', 3, 64)
 	val = strings.TrimRight(val, "0") // trim trailing zeros
 	val = strings.TrimRight(val, ".") // trim decimal point if trailing
 	val += " MiB"
@@ -1056,7 +1056,7 @@ func (s *Source) getAcceleratorCount(mfgID, devID string) (val string) {
 }
 
 func (s *Source) getAcceleratorQueues(accelName string) (val string) {
-	if accelName != "IAX" && accelName != "DSA" {
+	if accelName != "IAA" && accelName != "DSA" {
 		val = "N/A"
 		return
 	}
@@ -1068,3 +1068,12 @@ func (s *Source) getAcceleratorQueues(accelName string) (val string) {
 	val = strings.Join(lines, ", ")
 	return
 }
+
+func (s *Source) getVulnerabilities() (vulns map[string]string) {
+	vulns = make(map[string]string)
+	// from spectre-meltdown-checker
+	for _, pair := range s.valsArrayFromRegexSubmatch("spectre-meltdown-checker", `(CVE-\d+-\d+): (.+)`) {
+		vulns[pair[0]] = pair[1]
+	}
+	return
+}

version.txt

@@ -1 +1 @@
-2.7.0
+2.7.1