Skip to content

Commit

Permalink
Added optional dependency detection for npm lockfiles (version 2 and …
Browse files Browse the repository at this point in the history 
…3) (#1030)
  • Loading branch information
@RushabhBhansali
RushabhBhansali authored Mar 13, 2024
1 parent 5be8728 commit 0bbeeee
Show file tree
Hide file tree
Showing 8 changed files with 1,931 additions and 6 deletions.
4 changes: 2 additions & 2 deletions src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetectorWithRoots.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace Microsoft.ComponentDetection.Detectors.Npm;
namespace Microsoft.ComponentDetection.Detectors.Npm;

using System.Collections.Generic;
using System.Linq;
Expand Down Expand Up @@ -29,7 +29,7 @@ public NpmComponentDetectorWithRoots(IPathUtilityService pathUtilityService)

public override string Id => "NpmWithRoots";

public override int Version => 2;
public override int Version => 3;

protected override bool IsSupportedLockfileVersion(int lockfileVersion) => lockfileVersion != 3;

Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfile3Detector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public NpmLockfile3Detector(IPathUtilityService pathUtilityService)

public override string Id => "NpmLockfile3";

public override int Version => 1;
public override int Version => 2;

protected override bool IsSupportedLockfileVersion(int lockfileVersion) => lockfileVersion == 3;

Expand Down
3 changes: 2 additions & 1 deletion src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace Microsoft.ComponentDetection.Detectors.Npm;
namespace Microsoft.ComponentDetection.Detectors.Npm;

using System;
using System.Collections.Generic;
Expand Down Expand Up @@ -176,6 +176,7 @@ private void ProcessIndividualPackageJTokens(ISingleFileComponentRecorder single
var packageJsonToken = JToken.ReadFrom(reader);
var enqueued = this.TryEnqueueFirstLevelDependencies(topLevelDependencies, packageJsonToken["dependencies"], dependencyLookup, skipValidation: skipValidation);
enqueued = enqueued && this.TryEnqueueFirstLevelDependencies(topLevelDependencies, packageJsonToken["devDependencies"], dependencyLookup, skipValidation: skipValidation);
enqueued = enqueued && this.TryEnqueueFirstLevelDependencies(topLevelDependencies, packageJsonToken["optionalDependencies"], dependencyLookup, skipValidation: skipValidation);
if (!enqueued)
{
// This represents a mismatch between lock file and package.json, break out and do not register anything for these files
Expand Down
29 changes: 29 additions & 0 deletions test/Microsoft.ComponentDetection.Detectors.Tests/NpmDetectorWithRootsTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,35 @@ public async Task TestNpmDetector_PackageLockReturnsValidAsync()
}
}

[TestMethod]
public async Task TestNpmDetector_PackageLockReturnsValidWhenDevAndOptionalDependenciesAsync()
{
var rootName = Guid.NewGuid().ToString("N");
var rootVersion = NewRandomVersion();
var devDepName = Guid.NewGuid().ToString("N");
var devDepVersion = NewRandomVersion();
var optDepName = Guid.NewGuid().ToString("N");
var optDepVersion = NewRandomVersion();

var (packageLockName, packageLockContents, packageLockPath) = NpmTestUtilities.GetWellFormedPackageLock2WithOptionalAndDevDependency(this.packageLockJsonFileName, rootName, rootVersion, devDepName, devDepVersion, optDepName, optDepVersion);
var (packageJsonName, packageJsonContents, packageJsonPath) = NpmTestUtilities.GetPackageJsonOneRootOneDevDependencyOneOptionalDependency(rootName, rootVersion, devDepName, devDepVersion, optDepName, optDepVersion);

var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(packageLockName, packageLockContents, this.packageLockJsonSearchPatterns, fileLocation: packageLockPath)
.WithFile(packageJsonName, packageJsonContents, this.packageJsonSearchPattern, fileLocation: packageJsonPath)
.ExecuteDetectorAsync();

scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
var detectedComponents = componentRecorder.GetDetectedComponents();
detectedComponents.Should().HaveCount(2);

var retrievedDevDep = detectedComponents.Single(c => ((NpmComponent)c.Component).Name.Equals(devDepName));
componentRecorder.GetEffectiveDevDependencyValue(retrievedDevDep.Component.Id).Should().BeTrue();

var retrievedOptDep = detectedComponents.Single(c => ((NpmComponent)c.Component).Name.Equals(optDepName));
componentRecorder.GetEffectiveDevDependencyValue(retrievedOptDep.Component.Id).Should().BeFalse();
}

[TestMethod]
public async Task TestNpmDetector_MismatchedFilesReturnsEmptyAsync()
{
Expand Down
108 changes: 107 additions & 1 deletion test/Microsoft.ComponentDetection.Detectors.Tests/NpmLockfile3DetectorTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace Microsoft.ComponentDetection.Detectors.Tests;
namespace Microsoft.ComponentDetection.Detectors.Tests;

using System;
using System.Collections.Generic;
Expand Down Expand Up @@ -112,6 +112,112 @@ public async Task TestNpmDetector_PackageLockVersion3NestedReturnsValidAsync()
}
}

[TestMethod]
public async Task TestNpmDetector_PackageLockVersion3WithDevDependenciesReturnsValidAsync()
{
var componentName0 = Guid.NewGuid().ToString("N");
var version0 = NewRandomVersion();
var componentName1 = Guid.NewGuid().ToString("N");
var version1 = NewRandomVersion();

var (packageLockName, packageLockContents, packageLockPath) = NpmTestUtilities.GetWellFormedNestedPackageLock3WithDevDependencies(this.packageLockJsonFileName, componentName0, version0, componentName1, version1);

var packagejson = @"{{
""name"": ""test"",
""version"": ""0.0.0"",
""devDependencies"": {{
""{0}"": ""{1}"",
""{2}"": ""{3}""
}}
}}";

var packageJsonTemplate = string.Format(packagejson, componentName0, version0, componentName1, version1);

var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(packageLockName, packageLockContents, this.packageLockJsonSearchPatterns, fileLocation: packageLockPath)
.WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern)
.ExecuteDetectorAsync();

scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);

var detectedComponents = componentRecorder.GetDetectedComponents().ToList();
detectedComponents.Should().HaveCount(2);

var component0 = detectedComponents.First(x => x.Component.Id.Contains(componentName0));
componentRecorder.AssertAllExplicitlyReferencedComponents<NpmComponent>(
component0.Component.Id,
parentComponent0 => parentComponent0.Name == componentName0);

var component1 = detectedComponents.First(x => x.Component.Id.Contains(componentName1));
componentRecorder.GetEffectiveDevDependencyValue(component0.Component.Id).Should().BeTrue();
componentRecorder.AssertAllExplicitlyReferencedComponents<NpmComponent>(
component1.Component.Id,
parentComponent0 => parentComponent0.Name == componentName1);
componentRecorder.GetEffectiveDevDependencyValue(component1.Component.Id).Should().BeTrue();

foreach (var component in detectedComponents)
{
// check that either component0 or component1 is our parent
componentRecorder.IsDependencyOfExplicitlyReferencedComponents<NpmComponent>(
component.Component.Id,
parentComponent0 => parentComponent0.Name == componentName0 || parentComponent0.Name == componentName1);
((NpmComponent)component.Component).Hash.Should().NotBeNullOrWhiteSpace();
}
}

[TestMethod]
public async Task TestNpmDetector_PackageLockVersion3WithOptionalDependenciesReturnsValidAsync()
{
var componentName0 = Guid.NewGuid().ToString("N");
var version0 = NewRandomVersion();
var componentName1 = Guid.NewGuid().ToString("N");
var version1 = NewRandomVersion();

var (packageLockName, packageLockContents, packageLockPath) = NpmTestUtilities.GetWellFormedNestedPackageLock3WithOptionalDependencies(this.packageLockJsonFileName, componentName0, version0, componentName1, version1);

var packagejson = @"{{
""name"": ""test"",
""version"": ""0.0.0"",
""optionalDependencies"": {{
""{0}"": ""{1}"",
""{2}"": ""{3}""
}}
}}";

var packageJsonTemplate = string.Format(packagejson, componentName0, version0, componentName1, version1);

var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(packageLockName, packageLockContents, this.packageLockJsonSearchPatterns, fileLocation: packageLockPath)
.WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern)
.ExecuteDetectorAsync();

scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);

var detectedComponents = componentRecorder.GetDetectedComponents().ToList();
detectedComponents.Should().HaveCount(2);

var component0 = detectedComponents.First(x => x.Component.Id.Contains(componentName0));
componentRecorder.AssertAllExplicitlyReferencedComponents<NpmComponent>(
component0.Component.Id,
parentComponent0 => parentComponent0.Name == componentName0);

var component1 = detectedComponents.First(x => x.Component.Id.Contains(componentName1));
componentRecorder.GetEffectiveDevDependencyValue(component0.Component.Id).Should().BeFalse();
componentRecorder.AssertAllExplicitlyReferencedComponents<NpmComponent>(
component1.Component.Id,
parentComponent0 => parentComponent0.Name == componentName1);
componentRecorder.GetEffectiveDevDependencyValue(component1.Component.Id).Should().BeFalse();

foreach (var component in detectedComponents)
{
// check that either component0 or component1 is our parent
componentRecorder.IsDependencyOfExplicitlyReferencedComponents<NpmComponent>(
component.Component.Id,
parentComponent0 => parentComponent0.Name == componentName0 || parentComponent0.Name == componentName1);
((NpmComponent)component.Component).Hash.Should().NotBeNullOrWhiteSpace();
}
}

[TestMethod]
public async Task TestNpmDetector_NestedNodeModulesV3Async()
{
Expand Down
135 changes: 134 additions & 1 deletion test/Microsoft.ComponentDetection.Detectors.Tests/NpmTestUtilities.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace Microsoft.ComponentDetection.Detectors.Tests;
namespace Microsoft.ComponentDetection.Detectors.Tests;

using System;
using System.Collections.Generic;
Expand Down Expand Up @@ -88,6 +88,25 @@ public static (string PackageJsonName, string PackageJsonContents, string Packag
return ("package.json", packageJsonTemplate, Path.Combine(Path.GetTempPath(), "package.json"));
}

public static (string PackageJsonName, string PackageJsonContents, string PackageJsonPath) GetPackageJsonOneRootOneDevDependencyOneOptionalDependency(
string rootName, string rootVersion, string devDependencyName, string devDependencyVersion, string optionalDependencyName, string optionalDependencyVersion)
{
var packagejson = @"{{
""name"": ""{0}"",
""version"": ""{1}"",
""devDependencies"": {{
""{2}"": ""{3}""
}},
""optionalDependencies"": {{
""{4}"": ""{5}""
}}
}}";

var packageJsonTemplate = string.Format(packagejson, rootName, rootVersion, devDependencyName, devDependencyVersion, optionalDependencyName, optionalDependencyVersion);

return ("package.json", packageJsonTemplate, Path.Combine(Path.GetTempPath(), "package.json"));
}

public static (string PackageJsonName, string PackageJsonContents, string PackageJsonPath) GetPackageJsonNoDependenciesForNameAndVersion(string packageName, string packageVersion)
{
var packagejson = @"{{
Expand Down Expand Up @@ -238,6 +257,38 @@ public static (string PackageJsonName, string PackageJsonContents, string Packag
return (lockFileName, packageLockTemplate, Path.Combine(Path.GetTempPath(), lockFileName));
}

public static (string PackageJsonName, string PackageJsonContents, string PackageJsonPath) GetWellFormedPackageLock2WithOptionalAndDevDependency(
string lockFileName, string rootName = null, string rootVersion = null, string devDependencyName = null, string devDependencyVersion = null, string optionalDependencyName = null, string optionalDependencyVersion = null)
{
var packageLockJson = @"{{
""name"": ""{0}"",
""version"": ""{1}"",
""dependencies"": {{
""{2}"": {{
""version"": ""{3}"",
""resolved"": ""https://mseng.pkgs.visualstudio.com/_packaging/VsoMicrosoftExternals/npm/registry/"",
""integrity"": ""sha1-EBPRBRBH3TIP4k5JTVxm7K9hR9k="",
""dev"": true,
}},
""{4}"": {{
""version"": ""{5}"",
""resolved"": ""https://mseng.pkgs.visualstudio.com/_packaging/VsoMicrosoftExternals/npm/registry/"",
""integrity"": ""sha1-PRT306DRK/NZUaVL07iuqH7nWPg="",
""optional"": true,
}}
}}
}}";
rootName ??= Guid.NewGuid().ToString("N");
rootVersion ??= NewRandomVersion();
devDependencyName ??= Guid.NewGuid().ToString("N");
devDependencyVersion ??= NewRandomVersion();
optionalDependencyName ??= Guid.NewGuid().ToString("N");
optionalDependencyVersion ??= NewRandomVersion();
var packageLockTemplate = string.Format(packageLockJson, rootName, rootVersion, devDependencyName, devDependencyVersion, optionalDependencyName, optionalDependencyVersion);

return (lockFileName, packageLockTemplate, Path.Combine(Path.GetTempPath(), lockFileName));
}

public static (string PackageJsonName, string PackageJsonContents, string PackageJsonPath) GetWellFormedPackageLock3(string lockFileName, string rootName0 = null, string rootVersion0 = null, string rootName2 = null, string rootVersion2 = null, string packageName0 = "test", string packageName1 = null, string packageName3 = null)
{
var packageLockJson = @"{{
Expand Down Expand Up @@ -354,4 +405,86 @@ public static (string PackageJsonName, string PackageJsonContents, string Packag

return (lockFileName, packageLockTemplate, Path.Combine(Path.GetTempPath(), lockFileName));
}

public static (string PackageJsonName, string PackageJsonContents, string PackageJsonPath) GetWellFormedNestedPackageLock3WithDevDependencies(string lockFileName, string depName0 = null, string depVersion0 = null, string depName1 = null, string depVersion1 = null)
{
var packageLockJson = @"{{
""name"": ""test"",
""version"": ""0.0.0"",
""lockfileVersion"": 3,
""requires"": true,
""packages"": {{
"""": {{
""name"": ""test"",
""version"": ""0.0.0"",
""devDependencies"": {{
""{0}"": ""^{2}"",
""{1}"": ""^{3}""
}}
}},
""node_modules/{0}"": {{
""version"": ""{2}"",
""resolved"": ""https://mseng.pkgs.visualstudio.com/_packaging/VsoMicrosoftExternals/npm/registry"",
""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="",
""dev"": true
}},
""node_modules/{1}"": {{
""version"": ""{3}"",
""resolved"": ""https://mseng.pkgs.visualstudio.com/_packaging/VsoMicrosoftExternals/npm/registry"",
""integrity"": ""sha512-W86pkk7P9PAfARThHaD4fIjJ8QJUGMB2OhlCFsrueciPqlYZvDg/w62BmRm7PghVQcxGLbYoPN4+iykzP+0jRQ=="",
""dev"": true
}}
}}
}}";

var componentName0 = depName0 ?? Guid.NewGuid().ToString("N");
var version0 = depVersion0 ?? NewRandomVersion();
var componentName1 = depName1 ?? Guid.NewGuid().ToString("N");
var version1 = depVersion1 ?? NewRandomVersion();

var packageLockTemplate = string.Format(packageLockJson, componentName0, componentName1, version0, version1);

return (lockFileName, packageLockTemplate, Path.Combine(Path.GetTempPath(), lockFileName));
}

public static (string PackageJsonName, string PackageJsonContents, string PackageJsonPath) GetWellFormedNestedPackageLock3WithOptionalDependencies(string lockFileName, string depName0 = null, string depVersion0 = null, string depName1 = null, string depVersion1 = null)
{
var packageLockJson = @"{{
""name"": ""test"",
""version"": ""0.0.0"",
""lockfileVersion"": 3,
""requires"": true,
""packages"": {{
"""": {{
""name"": ""test"",
""version"": ""0.0.0"",
""optionalDependencies"": {{
""{0}"": ""^{2}"",
""{1}"": ""^{3}""
}}
}},
""node_modules/{0}"": {{
""version"": ""{2}"",
""resolved"": ""https://mseng.pkgs.visualstudio.com/_packaging/VsoMicrosoftExternals/npm/registry"",
""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="",
""optional"": true,
}},
""node_modules/{1}"": {{
""version"": ""{3}"",
""resolved"": ""https://mseng.pkgs.visualstudio.com/_packaging/VsoMicrosoftExternals/npm/registry"",
""integrity"": ""sha512-W86pkk7P9PAfARThHaD4fIjJ8QJUGMB2OhlCFsrueciPqlYZvDg/w62BmRm7PghVQcxGLbYoPN4+iykzP+0jRQ=="",
""optional"": true,
}}
}}
}}";

var componentName0 = depName0 ?? Guid.NewGuid().ToString("N");
var version0 = depVersion0 ?? NewRandomVersion();
var componentName1 = depName1 ?? Guid.NewGuid().ToString("N");
var version1 = depVersion1 ?? NewRandomVersion();

var packageLockTemplate = string.Format(packageLockJson, componentName0, componentName1, version0, version1);

return (lockFileName, packageLockTemplate, Path.Combine(Path.GetTempPath(), lockFileName));
}
}
Loading

0 comments on commit 0bbeeee

Please sign in to comment.