From ee00a55e13b065796bcfcc777451a17a52c0c661 Mon Sep 17 00:00:00 2001 From: Sarah Oslund Date: Fri, 23 Oct 2026 08:17:22 -0700 Subject: [PATCH 1/7] feat(spdx): add default-off SPDX 3 document detector Detect SPDX 3.0.0 and 3.0.1 compact JSON-LD documents as SBOM references, including root elements and creator metadata. Register the detector and add focused tests and a verification fixture. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 --- .../spdx/Spdx3ComponentDetector.cs | 197 ++++++++++++++++ .../Extensions/ServiceCollectionExtensions.cs | 1 + .../Spdx3ComponentDetectorTests.cs | 217 ++++++++++++++++++ .../resources/spdx3/manifest.spdx3.json | 45 ++++ 4 files changed, 460 insertions(+) create mode 100644 src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs create mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx3.json diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs new file mode 100644 index 000000000..1da2b7b03 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs @@ -0,0 +1,197 @@ +namespace Microsoft.ComponentDetection.Detectors.Spdx; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; + +public class Spdx3ComponentDetector : FileComponentDetector, IDefaultOffComponentDetector +{ + public Spdx3ComponentDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + IObservableDirectoryWalkerFactory walkerFactory, + ILogger logger) + { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.Logger = logger; + } + + public override string Id => "SPDX3SBOM"; + + public override IEnumerable Categories => [nameof(DetectorClass.Spdx)]; + + public override IEnumerable SupportedComponentTypes => [ComponentType.Spdx]; + + public override int Version => 1; + + public override IList SearchPatterns => ["*.spdx.json", "*.spdx3.json", "*.spdx.jsonld", "*.spdx3.jsonld"]; + + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) + { + var file = processRequest.ComponentStream; + this.Logger.LogDebug("Discovered SPDX 3 manifest file at: {ManifestLocation}", file.Location); + + try + { + // Keep the SHA1 document fingerprint used by SpdxComponent and existing SBOM reference consumers. +#pragma warning disable CA5350, CA1308 // Existing SPDX document references require lowercase SHA1 checksums. + var checksum = Convert.ToHexString(await SHA1.HashDataAsync(file.Stream, cancellationToken)).ToLowerInvariant(); +#pragma warning restore CA5350, CA1308 + file.Stream.Seek(0, SeekOrigin.Begin); + + using var json = await JsonDocument.ParseAsync(file.Stream, cancellationToken: cancellationToken); + var component = this.ReadDocument(json.RootElement, checksum, file.Location); + if (component != null) + { + processRequest.SingleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); + } + } + catch (JsonException e) + { + this.Logger.LogWarning(e, "Unable to parse SPDX 3 file at {ManifestLocation}, skipping", file.Location); + } + catch (IOException e) + { + this.Logger.LogError(e, "Error while reading SPDX 3 file at {ManifestLocation}", file.Location); + } + } + + private static string? ReadString(JsonElement element, string property) + { + if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) + { + return null; + } + + if (value.ValueKind != JsonValueKind.String) + { + throw new JsonException($"Expected a string for {property}."); + } + + return value.GetString(); + } + + private static bool HasType(JsonElement element, string type) + { + if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty("type", out var value)) + { + return false; + } + + return value.ValueKind == JsonValueKind.String + ? value.GetString() == type + : value.ValueKind == JsonValueKind.Array && value.EnumerateArray().Any(item => item.ValueKind == JsonValueKind.String && item.GetString() == type); + } + + private static string? ReadId(JsonElement element) => + element.ValueKind == JsonValueKind.String ? element.GetString() : ReadString(element, "spdxId") ?? ReadString(element, "@id"); + + private static JsonElement Resolve(JsonElement element, IDictionary elements) + { + var id = ReadId(element); + return id != null && elements.TryGetValue(id, out var resolved) ? resolved : element; + } + + private static IEnumerable ReadValues(JsonElement element, string property) + { + if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) + { + return []; + } + + return value.ValueKind == JsonValueKind.Array ? value.EnumerateArray().ToArray() : [value]; + } + + private static string? ReadCreator(JsonElement creationInfo, string property, string type, IDictionary elements) => + ReadValues(creationInfo, property) + .Select(element => Resolve(element, elements)) + .Where(element => HasType(element, type)) + .Select(element => ReadString(element, "name")) + .FirstOrDefault(name => !string.IsNullOrWhiteSpace(name)); + + private SpdxComponent? ReadDocument(JsonElement root, string checksum, string path) + { + if (root.ValueKind != JsonValueKind.Object + || !root.TryGetProperty("@graph", out var graph) + || graph.ValueKind != JsonValueKind.Array) + { + this.Logger.LogWarning("Discovered file at {ManifestLocation} is not an SPDX 3 JSON-LD graph, skipping", path); + return null; + } + + var documents = graph.EnumerateArray().Where(element => HasType(element, "SpdxDocument")).ToList(); + if (documents.Count != 1) + { + this.Logger.LogWarning("SPDX 3 file at {ManifestLocation} must contain exactly one SpdxDocument, skipping", path); + return null; + } + + var elements = new Dictionary(StringComparer.Ordinal); + foreach (var element in graph.EnumerateArray()) + { + var id = ReadId(element); + if (id != null && !elements.TryAdd(id, element)) + { + throw new JsonException($"Duplicate SPDX element identifier: {id}."); + } + } + + var document = documents[0]; + if (!document.TryGetProperty("creationInfo", out var creationInfo)) + { + throw new JsonException("SpdxDocument is missing creationInfo."); + } + + creationInfo = Resolve(creationInfo, elements); + var version = ReadString(creationInfo, "specVersion"); + if (version is not ("3.0.0" or "3.0.1")) + { + this.Logger.LogWarning("SPDX file at {ManifestLocation} has unsupported or missing specVersion {SpecVersion}, skipping", path, version); + return null; + } + + var documentId = ReadId(document); + if (string.IsNullOrWhiteSpace(documentId) || !Uri.TryCreate(documentId, UriKind.Absolute, out var documentUri)) + { + throw new JsonException("SpdxDocument must have an absolute spdxId."); + } + + var roots = ReadValues(document, "rootElement").Select(ReadId).ToList(); + if (roots.Any(id => string.IsNullOrWhiteSpace(id))) + { + throw new JsonException("SpdxDocument contains an invalid rootElement."); + } + + if (roots.Count > 1) + { + this.Logger.LogWarning("SPDX file at {ManifestLocation} has more than one rootElement, first will be selected as root element.", path); + } + else if (roots.Count == 0) + { + this.Logger.LogWarning("SPDX file at {ManifestLocation} has no rootElement, considering the SpdxDocument as the root element.", path); + } + + // SPDX 3 has no documentNamespace, and name is optional; the document IRI identifies the document. + var name = ReadString(document, "name"); + return new SpdxComponent( + $"SPDX-{version}", + documentUri, + string.IsNullOrWhiteSpace(name) ? documentId : name, + checksum, + roots.FirstOrDefault() ?? documentId, + path) + { + CreatorTool = ReadCreator(creationInfo, "createdUsing", "Tool", elements), + CreatorOrganization = ReadCreator(creationInfo, "createdBy", "Organization", elements), + }; + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index dd33c8a83..392186bad 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -160,6 +160,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s // SPDX services.AddSingleton(); + services.AddSingleton(); // VCPKG services.AddSingleton(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs new file mode 100644 index 000000000..6f371466b --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs @@ -0,0 +1,217 @@ +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using AwesomeAssertions; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Spdx; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class Spdx3ComponentDetectorTests : BaseDetectorTest +{ + private const string DocumentId = "https://example.org/sbom/document"; + private const string RootId = "https://example.org/sbom/package"; + private const string Manifest = """ + { + "@context": "https://spdx.org/rdf/3.0.1/spdx-context.jsonld", + "@graph": [ + { + "type": "SpdxDocument", + "spdxId": "https://example.org/sbom/document", + "name": "Example SBOM", + "creationInfo": "_:creation", + "rootElement": ["https://example.org/sbom/package"] + }, + { + "@id": "_:creation", + "type": "CreationInfo", + "specVersion": "3.0.1", + "created": "2026-01-01T00:00:00Z", + "createdBy": ["https://example.org/sbom/organization"], + "createdUsing": ["https://example.org/sbom/tool"] + }, + { + "type": "Organization", + "spdxId": "https://example.org/sbom/organization", + "name": "Example Organization", + "creationInfo": "_:creation" + }, + { + "type": "Tool", + "spdxId": "https://example.org/sbom/tool", + "name": "Example Tool", + "creationInfo": "_:creation" + }, + { + "type": "software_Package", + "spdxId": "https://example.org/sbom/package", + "name": "Example Package", + "creationInfo": "_:creation" + } + ] + } + """; + + [TestMethod] + [DataRow("3.0.0", "manifest.spdx.json")] + [DataRow("3.0.1", "manifest.spdx3.json")] + [DataRow("3.0.1", "manifest.spdx.jsonld")] + [DataRow("3.0.1", "manifest.spdx3.jsonld")] + public async Task DetectsDocumentWithoutImportingPackagesAsync(string version, string fileName) + { + var content = Manifest.Replace("3.0.1", version, StringComparison.Ordinal); + var (result, recorder) = await this.DetectorTestUtility + .WithFile(fileName, content) + .ExecuteDetectorAsync(); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + var component = recorder.GetDetectedComponents().Should().ContainSingle().Subject.Component + .Should().BeOfType().Subject; + +#pragma warning disable CA5350, CA1308 // Match the lowercase SHA1 fingerprint required by existing SPDX reference consumers. + var checksum = Convert.ToHexString(SHA1.HashData(Encoding.UTF8.GetBytes(content))).ToLowerInvariant(); +#pragma warning restore CA5350, CA1308 + + component.Name.Should().Be("Example SBOM"); + component.SpdxVersion.Should().Be($"SPDX-{version}"); + component.DocumentNamespace.Should().Be(new Uri(DocumentId)); + component.RootElementId.Should().Be(RootId); + component.Checksum.Should().Be(checksum); + component.Path.Should().Be(Path.Combine(Path.GetTempPath(), fileName)); + component.CreatorTool.Should().Be("Example Tool"); + component.CreatorOrganization.Should().Be("Example Organization"); + } + + [TestMethod] + public async Task ResolvesEmbeddedCreationInfoAndCreatorsAsync() + { + var json = JsonNode.Parse(Manifest)!; + var graph = json["@graph"]!.AsArray(); + var creation = graph[1]!.DeepClone(); + creation["createdBy"] = new JsonArray(graph[2]!.DeepClone()); + creation["createdUsing"] = new JsonArray(graph[3]!.DeepClone()); + graph[0]!["creationInfo"] = creation; + graph.RemoveAt(1); + + var (_, recorder) = await this.DetectorTestUtility + .WithFile("manifest.spdx3.json", json.ToJsonString()) + .ExecuteDetectorAsync(); + + var component = (SpdxComponent)recorder.GetDetectedComponents().Single().Component; + component.CreatorTool.Should().Be("Example Tool"); + component.CreatorOrganization.Should().Be("Example Organization"); + } + + [TestMethod] + [DataRow("[]", DocumentId)] + [DataRow("[\"https://example.org/sbom/package\", \"https://example.org/sbom/other\"]", RootId)] + [DataRow("\"https://example.org/sbom/package\"", RootId)] + [DataRow("[{\"spdxId\":\"https://example.org/sbom/package\"}]", RootId)] + public async Task SelectsRootAndUsesDocumentIdForUnnamedDocumentsAsync(string roots, string expectedRoot) + { + var json = JsonNode.Parse(Manifest)!; + var document = json["@graph"]![0]!.AsObject(); + document.Remove("name"); + document["rootElement"] = JsonNode.Parse(roots); + + var (_, recorder) = await this.DetectorTestUtility + .WithFile("manifest.spdx3.json", json.ToJsonString()) + .ExecuteDetectorAsync(); + + var component = (SpdxComponent)recorder.GetDetectedComponents().Single().Component; + component.Name.Should().Be(DocumentId); + component.RootElementId.Should().Be(expectedRoot); + } + + [TestMethod] + public async Task UsesDocumentAsRootWhenRootElementIsAbsentAsync() + { + var json = JsonNode.Parse(Manifest)!; + json["@graph"]![0]!.AsObject().Remove("rootElement"); + var creation = json["@graph"]![1]!.AsObject(); + creation.Remove("createdUsing"); + creation["createdBy"] = new JsonArray("https://example.org/unresolved-agent"); + + var (_, recorder) = await this.DetectorTestUtility + .WithFile("manifest.spdx3.json", json.ToJsonString()) + .ExecuteDetectorAsync(); + + var component = (SpdxComponent)recorder.GetDetectedComponents().Single().Component; + component.RootElementId.Should().Be(DocumentId); + component.CreatorTool.Should().BeNull(); + component.CreatorOrganization.Should().BeNull(); + } + + [TestMethod] + [DataRow("not json")] + [DataRow("[]")] + [DataRow("{}")] + [DataRow("{\"spdxVersion\":\"SPDX-2.2\"}")] + [DataRow("{\"@graph\":{}}")] + [DataRow("{\"@graph\":[]}")] + [DataRow("{\"@graph\":[{\"type\":\"software_Package\"}]}")] + public async Task SkipsNonSpdx3DocumentsWithWarningAsync(string content) + { + await this.AssertSkippedWithWarningAsync(content); + } + + [TestMethod] + [DataRow(0, "spdxId", "\"relative-document\"")] + [DataRow(0, "spdxId", "null")] + [DataRow(0, "creationInfo", "\"_:unresolved\"")] + [DataRow(0, "rootElement", "[42]")] + [DataRow(0, "name", "42")] + [DataRow(1, "specVersion", "\"2.2\"")] + [DataRow(1, "specVersion", "\"3.1.0\"")] + [DataRow(1, "specVersion", "\"3.0.1-draft\"")] + [DataRow(1, "specVersion", "null")] + public async Task SkipsInvalidOrUnsupportedDocumentsWithWarningAsync(int index, string property, string value) + { + var json = JsonNode.Parse(Manifest)!; + json["@graph"]![index]![property] = JsonNode.Parse(value); + await this.AssertSkippedWithWarningAsync(json.ToJsonString()); + } + + [TestMethod] + [DataRow(0)] + [DataRow(1)] + public async Task RejectsMultipleDocumentsOrDuplicateIdentifiersAsync(int index) + { + var json = JsonNode.Parse(Manifest)!; + var graph = json["@graph"]!.AsArray(); + graph.Add(graph[index]!.DeepClone()); + await this.AssertSkippedWithWarningAsync(json.ToJsonString()); + } + + private async Task AssertSkippedWithWarningAsync(string content) + { + var logger = new Mock>(); + var (result, recorder) = await this.DetectorTestUtility + .AddServiceMock(logger) + .WithFile("manifest.spdx3.json", content) + .ExecuteDetectorAsync(); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + recorder.GetDetectedComponents().Should().BeEmpty(); + logger.Verify( + log => log.Log( + LogLevel.Warning, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>()), + Times.AtLeastOnce); + } +} diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx3.json b/test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx3.json new file mode 100644 index 000000000..1d1f739cf --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx3.json @@ -0,0 +1,45 @@ +{ + "@context": "https://spdx.org/rdf/3.0.1/spdx-context.jsonld", + "@graph": [ + { + "@id": "_:creation", + "type": "CreationInfo", + "specVersion": "3.0.1", + "created": "2026-01-01T00:00:00Z", + "createdBy": ["https://example.org/spdx3/organization"], + "createdUsing": ["https://example.org/spdx3/tool"] + }, + { + "type": "Organization", + "spdxId": "https://example.org/spdx3/organization", + "name": "Example Organization", + "creationInfo": "_:creation" + }, + { + "type": "Tool", + "spdxId": "https://example.org/spdx3/tool", + "name": "Example SBOM Generator", + "creationInfo": "_:creation" + }, + { + "type": "SpdxDocument", + "spdxId": "https://example.org/spdx3/document", + "name": "SPDX 3 verification SBOM", + "creationInfo": "_:creation", + "profileConformance": ["core", "software"], + "rootElement": ["https://example.org/spdx3/package"], + "element": [ + "https://example.org/spdx3/organization", + "https://example.org/spdx3/tool", + "https://example.org/spdx3/package" + ] + }, + { + "type": "software_Package", + "spdxId": "https://example.org/spdx3/package", + "name": "Example Package", + "software_packageVersion": "1.0.0", + "creationInfo": "_:creation" + } + ] +} From 5b8ffacb78bc7ef9898ea21396343377835555b0 Mon Sep 17 00:00:00 2001 From: Sarah Oslund Date: Fri, 23 Oct 2026 08:28:40 -0700 Subject: [PATCH 2/7] fix(spdx): restrict SPDX 3 discovery to *.spdx.json Update detector tests and rename the verification fixture to match the sole supported search pattern. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 --- .../spdx/Spdx3ComponentDetector.cs | 2 +- .../Spdx3ComponentDetectorTests.cs | 17 ++++++++--------- .../{manifest.spdx3.json => manifest.spdx.json} | 0 3 files changed, 9 insertions(+), 10 deletions(-) rename test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/{manifest.spdx3.json => manifest.spdx.json} (100%) diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs index 1da2b7b03..613af88bb 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs @@ -33,7 +33,7 @@ public Spdx3ComponentDetector( public override int Version => 1; - public override IList SearchPatterns => ["*.spdx.json", "*.spdx3.json", "*.spdx.jsonld", "*.spdx3.jsonld"]; + public override IList SearchPatterns => ["*.spdx.json"]; protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs index 6f371466b..ecbbe03cc 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs @@ -65,12 +65,11 @@ public class Spdx3ComponentDetectorTests : BaseDetectorTest>(); var (result, recorder) = await this.DetectorTestUtility .AddServiceMock(logger) - .WithFile("manifest.spdx3.json", content) + .WithFile("manifest.spdx.json", content) .ExecuteDetectorAsync(); result.ResultCode.Should().Be(ProcessingResultCode.Success); diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx3.json b/test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx.json similarity index 100% rename from test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx3.json rename to test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx.json From a13acb381c1a0c8826b220225f6f5c67bf83d184 Mon Sep 17 00:00:00 2001 From: Sarah Oslund Date: Fri, 23 Oct 2026 08:36:48 -0700 Subject: [PATCH 3/7] refactor(spdx): scope the new detector to SPDX 3.0 Rename the detector, its ID, tests, and verification fixture to reserve distinct identities for future SPDX minor versions. Continue accepting SPDX 3.0.0 and 3.0.1 only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 --- ...tDetector.cs => Spdx30ComponentDetector.cs} | 18 +++++++++--------- .../Extensions/ServiceCollectionExtensions.cs | 2 +- ...ests.cs => Spdx30ComponentDetectorTests.cs} | 6 +++--- .../{spdx3 => spdx30}/manifest.spdx.json | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) rename src/Microsoft.ComponentDetection.Detectors/spdx/{Spdx3ComponentDetector.cs => Spdx30ComponentDetector.cs} (90%) rename test/Microsoft.ComponentDetection.Detectors.Tests/{Spdx3ComponentDetectorTests.cs => Spdx30ComponentDetectorTests.cs} (97%) rename test/Microsoft.ComponentDetection.VerificationTests/resources/{spdx3 => spdx30}/manifest.spdx.json (96%) diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs similarity index 90% rename from src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs rename to src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs index 613af88bb..c2a061a9e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx3ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs @@ -13,19 +13,19 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.Extensions.Logging; -public class Spdx3ComponentDetector : FileComponentDetector, IDefaultOffComponentDetector +public class Spdx30ComponentDetector : FileComponentDetector, IDefaultOffComponentDetector { - public Spdx3ComponentDetector( + public Spdx30ComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; this.Logger = logger; } - public override string Id => "SPDX3SBOM"; + public override string Id => "SPDX30SBOM"; public override IEnumerable Categories => [nameof(DetectorClass.Spdx)]; @@ -38,7 +38,7 @@ public Spdx3ComponentDetector( protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { var file = processRequest.ComponentStream; - this.Logger.LogDebug("Discovered SPDX 3 manifest file at: {ManifestLocation}", file.Location); + this.Logger.LogDebug("Discovered SPDX 3.0 manifest file at: {ManifestLocation}", file.Location); try { @@ -57,11 +57,11 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } catch (JsonException e) { - this.Logger.LogWarning(e, "Unable to parse SPDX 3 file at {ManifestLocation}, skipping", file.Location); + this.Logger.LogWarning(e, "Unable to parse SPDX 3.0 file at {ManifestLocation}, skipping", file.Location); } catch (IOException e) { - this.Logger.LogError(e, "Error while reading SPDX 3 file at {ManifestLocation}", file.Location); + this.Logger.LogError(e, "Error while reading SPDX 3.0 file at {ManifestLocation}", file.Location); } } @@ -124,14 +124,14 @@ private static IEnumerable ReadValues(JsonElement element, string p || !root.TryGetProperty("@graph", out var graph) || graph.ValueKind != JsonValueKind.Array) { - this.Logger.LogWarning("Discovered file at {ManifestLocation} is not an SPDX 3 JSON-LD graph, skipping", path); + this.Logger.LogWarning("Discovered file at {ManifestLocation} is not an SPDX 3.0 JSON-LD graph, skipping", path); return null; } var documents = graph.EnumerateArray().Where(element => HasType(element, "SpdxDocument")).ToList(); if (documents.Count != 1) { - this.Logger.LogWarning("SPDX 3 file at {ManifestLocation} must contain exactly one SpdxDocument, skipping", path); + this.Logger.LogWarning("SPDX 3.0 file at {ManifestLocation} must contain exactly one SpdxDocument, skipping", path); return null; } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index 392186bad..8e7eaa8ef 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -160,7 +160,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s // SPDX services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); // VCPKG services.AddSingleton(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs similarity index 97% rename from test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs rename to test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs index ecbbe03cc..c06cf0eb9 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx3ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs @@ -19,7 +19,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; [TestClass] [TestCategory("Governance/All")] [TestCategory("Governance/ComponentDetection")] -public class Spdx3ComponentDetectorTests : BaseDetectorTest +public class Spdx30ComponentDetectorTests : BaseDetectorTest { private const string DocumentId = "https://example.org/sbom/document"; private const string RootId = "https://example.org/sbom/package"; @@ -161,7 +161,7 @@ public async Task UsesDocumentAsRootWhenRootElementIsAbsentAsync() [DataRow("{\"@graph\":{}}")] [DataRow("{\"@graph\":[]}")] [DataRow("{\"@graph\":[{\"type\":\"software_Package\"}]}")] - public async Task SkipsNonSpdx3DocumentsWithWarningAsync(string content) + public async Task SkipsNonSpdx30DocumentsWithWarningAsync(string content) { await this.AssertSkippedWithWarningAsync(content); } @@ -196,7 +196,7 @@ public async Task RejectsMultipleDocumentsOrDuplicateIdentifiersAsync(int index) private async Task AssertSkippedWithWarningAsync(string content) { - var logger = new Mock>(); + var logger = new Mock>(); var (result, recorder) = await this.DetectorTestUtility .AddServiceMock(logger) .WithFile("manifest.spdx.json", content) diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx.json b/test/Microsoft.ComponentDetection.VerificationTests/resources/spdx30/manifest.spdx.json similarity index 96% rename from test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx.json rename to test/Microsoft.ComponentDetection.VerificationTests/resources/spdx30/manifest.spdx.json index 1d1f739cf..7eceb5063 100644 --- a/test/Microsoft.ComponentDetection.VerificationTests/resources/spdx3/manifest.spdx.json +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/spdx30/manifest.spdx.json @@ -24,7 +24,7 @@ { "type": "SpdxDocument", "spdxId": "https://example.org/spdx3/document", - "name": "SPDX 3 verification SBOM", + "name": "SPDX 3.0 verification SBOM", "creationInfo": "_:creation", "profileConformance": ["core", "software"], "rootElement": ["https://example.org/spdx3/package"], From b7caee81b61574adcb73f4ecae7e4b9b9a09eb9d Mon Sep 17 00:00:00 2001 From: Sarah Oslund Date: Fri, 23 Oct 2026 09:00:12 -0700 Subject: [PATCH 4/7] refactor(spdx): share common detector file processing Extract shared SPDX metadata, SHA1 hashing, JSON loading, and component registration into SpdxComponentDetector. Keep format-specific parsing and exception handling in the concrete detectors, with regression coverage for their differing error policies. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 --- .../spdx/Spdx22ComponentDetector.cs | 70 +++------ .../spdx/Spdx30ComponentDetector.cs | 134 ++++++++---------- .../spdx/SpdxComponentDetector.cs | 52 +++++++ .../SPDX22ComponentDetectorTests.cs | 30 ++++ .../Spdx30ComponentDetectorTests.cs | 16 +++ 5 files changed, 176 insertions(+), 126 deletions(-) create mode 100644 src/Microsoft.ComponentDetection.Detectors/spdx/SpdxComponentDetector.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs index 77de63730..c95e728c8 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs @@ -3,9 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; using System; using System.Collections.Generic; -using System.IO; using System.Linq; -using System.Security.Cryptography; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -18,7 +16,7 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; /// Spdx22ComponentDetector discover SPDX SBOM files in JSON format and create components with the information about /// what SPDX document describes. /// -public class Spdx22ComponentDetector : FileComponentDetector, IDefaultOffComponentDetector +public class Spdx22ComponentDetector : SpdxComponentDetector, IDefaultOffComponentDetector { private readonly IEnumerable supportedSPDXVersions = ["SPDX-2.2"]; @@ -26,54 +24,25 @@ public Spdx22ComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, ILogger logger) + : base(componentStreamEnumerableFactory, walkerFactory, logger) { - this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; - this.Scanner = walkerFactory; - this.Logger = logger; } - public override IEnumerable Categories => - [Enum.GetName(typeof(DetectorClass), DetectorClass.Spdx)]; - public override string Id => "SPDX22SBOM"; - public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Spdx]; - public override int Version => 1; - public override IList SearchPatterns => ["*.spdx.json"]; - protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { this.Logger.LogDebug("Discovered SPDX2.2 manifest file at: {ManifestLocation}", processRequest.ComponentStream.Location); - var file = processRequest.ComponentStream; try { - var hash = this.GetSHA1HashFromStream(file.Stream); - - // Reset buffer to starting position after hash generation. - file.Stream.Seek(0, SeekOrigin.Begin); - - try - { - using var document = await JsonDocument.ParseAsync(file.Stream, cancellationToken: cancellationToken).ConfigureAwait(false); - var root = document.RootElement; - - if (this.IsSPDXVersionSupported(root)) - { - var sbomComponent = this.ConvertJsonElementToSbomComponent(processRequest, root, hash); - processRequest.SingleFileComponentRecorder.RegisterUsage(new DetectedComponent(sbomComponent)); - } - else - { - this.Logger.LogWarning("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", processRequest.ComponentStream.Location); - } - } - catch (JsonException) - { - this.Logger.LogWarning("Unable to parse file at {ManifestLocation}, skipping", processRequest.ComponentStream.Location); - } + await this.ProcessFileAsync(processRequest, cancellationToken).ConfigureAwait(false); + } + catch (JsonException) + { + this.Logger.LogWarning("Unable to parse file at {ManifestLocation}, skipping", processRequest.ComponentStream.Location); } catch (Exception e) { @@ -81,6 +50,17 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } } + protected override SpdxComponent ReadDocument(JsonElement root, string checksum, string path) + { + if (this.IsSPDXVersionSupported(root)) + { + return this.ConvertJsonElementToSbomComponent(root, checksum, path); + } + + this.Logger.LogWarning("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", path); + return null; + } + private bool IsSPDXVersionSupported(JsonElement document) { if (document.TryGetProperty("spdxVersion", out var versionElement)) @@ -92,7 +72,7 @@ private bool IsSPDXVersionSupported(JsonElement document) return false; } - private SpdxComponent ConvertJsonElementToSbomComponent(ProcessRequest processRequest, JsonElement document, string fileHash) + private SpdxComponent ConvertJsonElementToSbomComponent(JsonElement document, string fileHash, string path) { var sbomNamespace = document.TryGetProperty("documentNamespace", out var nsElement) ? nsElement.GetString() : null; var name = document.TryGetProperty("name", out var nameElement) ? nameElement.GetString() : null; @@ -109,16 +89,15 @@ private SpdxComponent ConvertJsonElementToSbomComponent(ProcessRequest processRe if (rootElements is { Length: > 1 }) { - this.Logger.LogWarning("SPDX file at {ManifestLocation} has more than one element in documentDescribes, first will be selected as root element.", processRequest.ComponentStream.Location); + this.Logger.LogWarning("SPDX file at {ManifestLocation} has more than one element in documentDescribes, first will be selected as root element.", path); } if (rootElements is { Length: 0 }) { - this.Logger.LogWarning("SPDX file at {ManifestLocation} does not have root elements in documentDescribes section, considering SPDXRef-Document as a root element.", processRequest.ComponentStream.Location); + this.Logger.LogWarning("SPDX file at {ManifestLocation} does not have root elements in documentDescribes section, considering SPDXRef-Document as a root element.", path); } var rootElementId = rootElements?.FirstOrDefault() ?? "SPDXRef-Document"; - var path = processRequest.ComponentStream.Location; var component = new SpdxComponent(spdxVersion, new Uri(sbomNamespace), name, fileHash, rootElementId, path); if (document.TryGetProperty("creationInfo", out var creationInfoElement) @@ -159,11 +138,4 @@ private SpdxComponent ConvertJsonElementToSbomComponent(ProcessRequest processRe return component; } - - private string GetSHA1HashFromStream(Stream stream) - { -#pragma warning disable CA5350 // Suppress Do Not Use Weak Cryptographic Algorithms because we use SHA1 intentionally in SPDX format - return BitConverter.ToString(SHA1.Create().ComputeHash(stream)).Replace("-", string.Empty).ToLower(); // CodeQL [SM02196] Sha1 is used in SPDX 2.2 format this file is parsing (https://spdx.github.io/spdx-spec/v2.2.2/file-information/). -#pragma warning restore CA5350 - } } diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs index c2a061a9e..073b951d3 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs @@ -4,7 +4,6 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Security.Cryptography; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -13,28 +12,20 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.Extensions.Logging; -public class Spdx30ComponentDetector : FileComponentDetector, IDefaultOffComponentDetector +public class Spdx30ComponentDetector : SpdxComponentDetector, IDefaultOffComponentDetector { public Spdx30ComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, ILogger logger) + : base(componentStreamEnumerableFactory, walkerFactory, logger) { - this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; - this.Scanner = walkerFactory; - this.Logger = logger; } public override string Id => "SPDX30SBOM"; - public override IEnumerable Categories => [nameof(DetectorClass.Spdx)]; - - public override IEnumerable SupportedComponentTypes => [ComponentType.Spdx]; - public override int Version => 1; - public override IList SearchPatterns => ["*.spdx.json"]; - protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { var file = processRequest.ComponentStream; @@ -42,18 +33,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID try { - // Keep the SHA1 document fingerprint used by SpdxComponent and existing SBOM reference consumers. -#pragma warning disable CA5350, CA1308 // Existing SPDX document references require lowercase SHA1 checksums. - var checksum = Convert.ToHexString(await SHA1.HashDataAsync(file.Stream, cancellationToken)).ToLowerInvariant(); -#pragma warning restore CA5350, CA1308 - file.Stream.Seek(0, SeekOrigin.Begin); - - using var json = await JsonDocument.ParseAsync(file.Stream, cancellationToken: cancellationToken); - var component = this.ReadDocument(json.RootElement, checksum, file.Location); - if (component != null) - { - processRequest.SingleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); - } + await this.ProcessFileAsync(processRequest, cancellationToken); } catch (JsonException e) { @@ -65,60 +45,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } } - private static string? ReadString(JsonElement element, string property) - { - if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) - { - return null; - } - - if (value.ValueKind != JsonValueKind.String) - { - throw new JsonException($"Expected a string for {property}."); - } - - return value.GetString(); - } - - private static bool HasType(JsonElement element, string type) - { - if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty("type", out var value)) - { - return false; - } - - return value.ValueKind == JsonValueKind.String - ? value.GetString() == type - : value.ValueKind == JsonValueKind.Array && value.EnumerateArray().Any(item => item.ValueKind == JsonValueKind.String && item.GetString() == type); - } - - private static string? ReadId(JsonElement element) => - element.ValueKind == JsonValueKind.String ? element.GetString() : ReadString(element, "spdxId") ?? ReadString(element, "@id"); - - private static JsonElement Resolve(JsonElement element, IDictionary elements) - { - var id = ReadId(element); - return id != null && elements.TryGetValue(id, out var resolved) ? resolved : element; - } - - private static IEnumerable ReadValues(JsonElement element, string property) - { - if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) - { - return []; - } - - return value.ValueKind == JsonValueKind.Array ? value.EnumerateArray().ToArray() : [value]; - } - - private static string? ReadCreator(JsonElement creationInfo, string property, string type, IDictionary elements) => - ReadValues(creationInfo, property) - .Select(element => Resolve(element, elements)) - .Where(element => HasType(element, type)) - .Select(element => ReadString(element, "name")) - .FirstOrDefault(name => !string.IsNullOrWhiteSpace(name)); - - private SpdxComponent? ReadDocument(JsonElement root, string checksum, string path) + protected override SpdxComponent? ReadDocument(JsonElement root, string checksum, string path) { if (root.ValueKind != JsonValueKind.Object || !root.TryGetProperty("@graph", out var graph) @@ -194,4 +121,57 @@ private static IEnumerable ReadValues(JsonElement element, string p CreatorOrganization = ReadCreator(creationInfo, "createdBy", "Organization", elements), }; } + + private static string? ReadString(JsonElement element, string property) + { + if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) + { + return null; + } + + if (value.ValueKind != JsonValueKind.String) + { + throw new JsonException($"Expected a string for {property}."); + } + + return value.GetString(); + } + + private static bool HasType(JsonElement element, string type) + { + if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty("type", out var value)) + { + return false; + } + + return value.ValueKind == JsonValueKind.String + ? value.GetString() == type + : value.ValueKind == JsonValueKind.Array && value.EnumerateArray().Any(item => item.ValueKind == JsonValueKind.String && item.GetString() == type); + } + + private static string? ReadId(JsonElement element) => + element.ValueKind == JsonValueKind.String ? element.GetString() : ReadString(element, "spdxId") ?? ReadString(element, "@id"); + + private static JsonElement Resolve(JsonElement element, IDictionary elements) + { + var id = ReadId(element); + return id != null && elements.TryGetValue(id, out var resolved) ? resolved : element; + } + + private static IEnumerable ReadValues(JsonElement element, string property) + { + if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) + { + return []; + } + + return value.ValueKind == JsonValueKind.Array ? value.EnumerateArray().ToArray() : [value]; + } + + private static string? ReadCreator(JsonElement creationInfo, string property, string type, IDictionary elements) => + ReadValues(creationInfo, property) + .Select(element => Resolve(element, elements)) + .Where(element => HasType(element, type)) + .Select(element => ReadString(element, "name")) + .FirstOrDefault(name => !string.IsNullOrWhiteSpace(name)); } diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/SpdxComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/SpdxComponentDetector.cs new file mode 100644 index 000000000..ac63a6d08 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/SpdxComponentDetector.cs @@ -0,0 +1,52 @@ +namespace Microsoft.ComponentDetection.Detectors.Spdx; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; + +public abstract class SpdxComponentDetector : FileComponentDetector +{ + protected SpdxComponentDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + IObservableDirectoryWalkerFactory walkerFactory, + ILogger logger) + { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.Logger = logger; + } + + public override IEnumerable Categories => [nameof(DetectorClass.Spdx)]; + + public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Spdx]; + + public override IList SearchPatterns => ["*.spdx.json"]; + + protected abstract SpdxComponent? ReadDocument(JsonElement root, string checksum, string path); + + protected async Task ProcessFileAsync(ProcessRequest processRequest, CancellationToken cancellationToken) + { + var file = processRequest.ComponentStream; + using var stream = file.Stream; + +#pragma warning disable CA5350, CA1308 // Existing SPDX document references require lowercase SHA1 checksums. + var checksum = Convert.ToHexString(await SHA1.HashDataAsync(stream, cancellationToken)).ToLowerInvariant(); +#pragma warning restore CA5350, CA1308 + stream.Seek(0, SeekOrigin.Begin); + + using var json = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); + var component = this.ReadDocument(json.RootElement, checksum, file.Location); + if (component != null) + { + processRequest.SingleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); + } + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs index c53bfedb2..15ce034a6 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs @@ -7,6 +7,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using System.Linq; using System.Security.Cryptography; using System.Text; +using System.Threading; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.ComponentDetection.Common.DependencyGraph; @@ -14,7 +15,9 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Spdx; using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; [TestClass] [TestCategory("Governance/All")] @@ -164,6 +167,33 @@ public async Task TestSbomDetector_InvalidFileAsync() components.Should().BeEmpty(); } + [TestMethod] + public async Task TestSbomDetector_LogsUnexpectedStreamErrorsAsync() + { + var error = new InvalidOperationException("Read failed."); + var stream = new Mock(); + stream.SetupGet(s => s.CanRead).Returns(true); + stream.Setup(s => s.ReadAsync(It.IsAny>(), It.IsAny())) + .ThrowsAsync(error); + var logger = new Mock>(); + + var (result, recorder) = await this.detectorTestUtility + .AddServiceMock(logger) + .WithFile("manifest.spdx.json", stream.Object) + .ExecuteDetectorAsync(); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + recorder.GetDetectedComponents().Should().BeEmpty(); + logger.Verify( + log => log.Log( + LogLevel.Error, + It.IsAny(), + It.IsAny(), + error, + It.IsAny>()), + Times.Once); + } + [TestMethod] public async Task TestSbomDetector_ExtractsCreatorToolAndOrganizationAsync() { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs index c06cf0eb9..ec0c4cbab 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs @@ -6,6 +6,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using System.Security.Cryptography; using System.Text; using System.Text.Json.Nodes; +using System.Threading; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.ComponentDetection.Contracts; @@ -194,6 +195,21 @@ public async Task RejectsMultipleDocumentsOrDuplicateIdentifiersAsync(int index) await this.AssertSkippedWithWarningAsync(json.ToJsonString()); } + [TestMethod] + public async Task PropagatesUnexpectedStreamErrorsAsync() + { + var stream = new Mock(); + stream.SetupGet(s => s.CanRead).Returns(true); + stream.Setup(s => s.ReadAsync(It.IsAny>(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("Read failed.")); + + Func action = () => this.DetectorTestUtility + .WithFile("manifest.spdx.json", stream.Object) + .ExecuteDetectorAsync(); + + await action.Should().ThrowAsync().WithMessage("Read failed."); + } + private async Task AssertSkippedWithWarningAsync(string content) { var logger = new Mock>(); From 7ea065f1d14a829f08c4ebf75ea5ade369912666 Mon Sep 17 00:00:00 2001 From: Sarah Oslund Date: Fri, 23 Oct 2026 09:08:10 -0700 Subject: [PATCH 5/7] Revert "refactor(spdx): share common detector file processing" This reverts commit b7caee81b61574adcb73f4ecae7e4b9b9a09eb9d, restoring independent SPDX detector implementations. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 74ac0491-574a-40fc-bf08-180aea181b07 --- .../spdx/Spdx22ComponentDetector.cs | 70 ++++++--- .../spdx/Spdx30ComponentDetector.cs | 134 ++++++++++-------- .../spdx/SpdxComponentDetector.cs | 52 ------- .../SPDX22ComponentDetectorTests.cs | 30 ---- .../Spdx30ComponentDetectorTests.cs | 16 --- 5 files changed, 126 insertions(+), 176 deletions(-) delete mode 100644 src/Microsoft.ComponentDetection.Detectors/spdx/SpdxComponentDetector.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs index c95e728c8..77de63730 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs @@ -3,7 +3,9 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -16,7 +18,7 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; /// Spdx22ComponentDetector discover SPDX SBOM files in JSON format and create components with the information about /// what SPDX document describes. /// -public class Spdx22ComponentDetector : SpdxComponentDetector, IDefaultOffComponentDetector +public class Spdx22ComponentDetector : FileComponentDetector, IDefaultOffComponentDetector { private readonly IEnumerable supportedSPDXVersions = ["SPDX-2.2"]; @@ -24,25 +26,54 @@ public Spdx22ComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, ILogger logger) - : base(componentStreamEnumerableFactory, walkerFactory, logger) { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.Logger = logger; } + public override IEnumerable Categories => + [Enum.GetName(typeof(DetectorClass), DetectorClass.Spdx)]; + public override string Id => "SPDX22SBOM"; + public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Spdx]; + public override int Version => 1; + public override IList SearchPatterns => ["*.spdx.json"]; + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { this.Logger.LogDebug("Discovered SPDX2.2 manifest file at: {ManifestLocation}", processRequest.ComponentStream.Location); + var file = processRequest.ComponentStream; try { - await this.ProcessFileAsync(processRequest, cancellationToken).ConfigureAwait(false); - } - catch (JsonException) - { - this.Logger.LogWarning("Unable to parse file at {ManifestLocation}, skipping", processRequest.ComponentStream.Location); + var hash = this.GetSHA1HashFromStream(file.Stream); + + // Reset buffer to starting position after hash generation. + file.Stream.Seek(0, SeekOrigin.Begin); + + try + { + using var document = await JsonDocument.ParseAsync(file.Stream, cancellationToken: cancellationToken).ConfigureAwait(false); + var root = document.RootElement; + + if (this.IsSPDXVersionSupported(root)) + { + var sbomComponent = this.ConvertJsonElementToSbomComponent(processRequest, root, hash); + processRequest.SingleFileComponentRecorder.RegisterUsage(new DetectedComponent(sbomComponent)); + } + else + { + this.Logger.LogWarning("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", processRequest.ComponentStream.Location); + } + } + catch (JsonException) + { + this.Logger.LogWarning("Unable to parse file at {ManifestLocation}, skipping", processRequest.ComponentStream.Location); + } } catch (Exception e) { @@ -50,17 +81,6 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } } - protected override SpdxComponent ReadDocument(JsonElement root, string checksum, string path) - { - if (this.IsSPDXVersionSupported(root)) - { - return this.ConvertJsonElementToSbomComponent(root, checksum, path); - } - - this.Logger.LogWarning("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", path); - return null; - } - private bool IsSPDXVersionSupported(JsonElement document) { if (document.TryGetProperty("spdxVersion", out var versionElement)) @@ -72,7 +92,7 @@ private bool IsSPDXVersionSupported(JsonElement document) return false; } - private SpdxComponent ConvertJsonElementToSbomComponent(JsonElement document, string fileHash, string path) + private SpdxComponent ConvertJsonElementToSbomComponent(ProcessRequest processRequest, JsonElement document, string fileHash) { var sbomNamespace = document.TryGetProperty("documentNamespace", out var nsElement) ? nsElement.GetString() : null; var name = document.TryGetProperty("name", out var nameElement) ? nameElement.GetString() : null; @@ -89,15 +109,16 @@ private SpdxComponent ConvertJsonElementToSbomComponent(JsonElement document, st if (rootElements is { Length: > 1 }) { - this.Logger.LogWarning("SPDX file at {ManifestLocation} has more than one element in documentDescribes, first will be selected as root element.", path); + this.Logger.LogWarning("SPDX file at {ManifestLocation} has more than one element in documentDescribes, first will be selected as root element.", processRequest.ComponentStream.Location); } if (rootElements is { Length: 0 }) { - this.Logger.LogWarning("SPDX file at {ManifestLocation} does not have root elements in documentDescribes section, considering SPDXRef-Document as a root element.", path); + this.Logger.LogWarning("SPDX file at {ManifestLocation} does not have root elements in documentDescribes section, considering SPDXRef-Document as a root element.", processRequest.ComponentStream.Location); } var rootElementId = rootElements?.FirstOrDefault() ?? "SPDXRef-Document"; + var path = processRequest.ComponentStream.Location; var component = new SpdxComponent(spdxVersion, new Uri(sbomNamespace), name, fileHash, rootElementId, path); if (document.TryGetProperty("creationInfo", out var creationInfoElement) @@ -138,4 +159,11 @@ private SpdxComponent ConvertJsonElementToSbomComponent(JsonElement document, st return component; } + + private string GetSHA1HashFromStream(Stream stream) + { +#pragma warning disable CA5350 // Suppress Do Not Use Weak Cryptographic Algorithms because we use SHA1 intentionally in SPDX format + return BitConverter.ToString(SHA1.Create().ComputeHash(stream)).Replace("-", string.Empty).ToLower(); // CodeQL [SM02196] Sha1 is used in SPDX 2.2 format this file is parsing (https://spdx.github.io/spdx-spec/v2.2.2/file-information/). +#pragma warning restore CA5350 + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs index 073b951d3..c2a061a9e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs @@ -4,6 +4,7 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -12,20 +13,28 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.Extensions.Logging; -public class Spdx30ComponentDetector : SpdxComponentDetector, IDefaultOffComponentDetector +public class Spdx30ComponentDetector : FileComponentDetector, IDefaultOffComponentDetector { public Spdx30ComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, ILogger logger) - : base(componentStreamEnumerableFactory, walkerFactory, logger) { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.Logger = logger; } public override string Id => "SPDX30SBOM"; + public override IEnumerable Categories => [nameof(DetectorClass.Spdx)]; + + public override IEnumerable SupportedComponentTypes => [ComponentType.Spdx]; + public override int Version => 1; + public override IList SearchPatterns => ["*.spdx.json"]; + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { var file = processRequest.ComponentStream; @@ -33,7 +42,18 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID try { - await this.ProcessFileAsync(processRequest, cancellationToken); + // Keep the SHA1 document fingerprint used by SpdxComponent and existing SBOM reference consumers. +#pragma warning disable CA5350, CA1308 // Existing SPDX document references require lowercase SHA1 checksums. + var checksum = Convert.ToHexString(await SHA1.HashDataAsync(file.Stream, cancellationToken)).ToLowerInvariant(); +#pragma warning restore CA5350, CA1308 + file.Stream.Seek(0, SeekOrigin.Begin); + + using var json = await JsonDocument.ParseAsync(file.Stream, cancellationToken: cancellationToken); + var component = this.ReadDocument(json.RootElement, checksum, file.Location); + if (component != null) + { + processRequest.SingleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); + } } catch (JsonException e) { @@ -45,7 +65,60 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } } - protected override SpdxComponent? ReadDocument(JsonElement root, string checksum, string path) + private static string? ReadString(JsonElement element, string property) + { + if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) + { + return null; + } + + if (value.ValueKind != JsonValueKind.String) + { + throw new JsonException($"Expected a string for {property}."); + } + + return value.GetString(); + } + + private static bool HasType(JsonElement element, string type) + { + if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty("type", out var value)) + { + return false; + } + + return value.ValueKind == JsonValueKind.String + ? value.GetString() == type + : value.ValueKind == JsonValueKind.Array && value.EnumerateArray().Any(item => item.ValueKind == JsonValueKind.String && item.GetString() == type); + } + + private static string? ReadId(JsonElement element) => + element.ValueKind == JsonValueKind.String ? element.GetString() : ReadString(element, "spdxId") ?? ReadString(element, "@id"); + + private static JsonElement Resolve(JsonElement element, IDictionary elements) + { + var id = ReadId(element); + return id != null && elements.TryGetValue(id, out var resolved) ? resolved : element; + } + + private static IEnumerable ReadValues(JsonElement element, string property) + { + if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) + { + return []; + } + + return value.ValueKind == JsonValueKind.Array ? value.EnumerateArray().ToArray() : [value]; + } + + private static string? ReadCreator(JsonElement creationInfo, string property, string type, IDictionary elements) => + ReadValues(creationInfo, property) + .Select(element => Resolve(element, elements)) + .Where(element => HasType(element, type)) + .Select(element => ReadString(element, "name")) + .FirstOrDefault(name => !string.IsNullOrWhiteSpace(name)); + + private SpdxComponent? ReadDocument(JsonElement root, string checksum, string path) { if (root.ValueKind != JsonValueKind.Object || !root.TryGetProperty("@graph", out var graph) @@ -121,57 +194,4 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID CreatorOrganization = ReadCreator(creationInfo, "createdBy", "Organization", elements), }; } - - private static string? ReadString(JsonElement element, string property) - { - if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) - { - return null; - } - - if (value.ValueKind != JsonValueKind.String) - { - throw new JsonException($"Expected a string for {property}."); - } - - return value.GetString(); - } - - private static bool HasType(JsonElement element, string type) - { - if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty("type", out var value)) - { - return false; - } - - return value.ValueKind == JsonValueKind.String - ? value.GetString() == type - : value.ValueKind == JsonValueKind.Array && value.EnumerateArray().Any(item => item.ValueKind == JsonValueKind.String && item.GetString() == type); - } - - private static string? ReadId(JsonElement element) => - element.ValueKind == JsonValueKind.String ? element.GetString() : ReadString(element, "spdxId") ?? ReadString(element, "@id"); - - private static JsonElement Resolve(JsonElement element, IDictionary elements) - { - var id = ReadId(element); - return id != null && elements.TryGetValue(id, out var resolved) ? resolved : element; - } - - private static IEnumerable ReadValues(JsonElement element, string property) - { - if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) - { - return []; - } - - return value.ValueKind == JsonValueKind.Array ? value.EnumerateArray().ToArray() : [value]; - } - - private static string? ReadCreator(JsonElement creationInfo, string property, string type, IDictionary elements) => - ReadValues(creationInfo, property) - .Select(element => Resolve(element, elements)) - .Where(element => HasType(element, type)) - .Select(element => ReadString(element, "name")) - .FirstOrDefault(name => !string.IsNullOrWhiteSpace(name)); } diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/SpdxComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/SpdxComponentDetector.cs deleted file mode 100644 index ac63a6d08..000000000 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/SpdxComponentDetector.cs +++ /dev/null @@ -1,52 +0,0 @@ -namespace Microsoft.ComponentDetection.Detectors.Spdx; - -using System; -using System.Collections.Generic; -using System.IO; -using System.Security.Cryptography; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.ComponentDetection.Contracts; -using Microsoft.ComponentDetection.Contracts.Internal; -using Microsoft.ComponentDetection.Contracts.TypedComponent; -using Microsoft.Extensions.Logging; - -public abstract class SpdxComponentDetector : FileComponentDetector -{ - protected SpdxComponentDetector( - IComponentStreamEnumerableFactory componentStreamEnumerableFactory, - IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) - { - this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; - this.Scanner = walkerFactory; - this.Logger = logger; - } - - public override IEnumerable Categories => [nameof(DetectorClass.Spdx)]; - - public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Spdx]; - - public override IList SearchPatterns => ["*.spdx.json"]; - - protected abstract SpdxComponent? ReadDocument(JsonElement root, string checksum, string path); - - protected async Task ProcessFileAsync(ProcessRequest processRequest, CancellationToken cancellationToken) - { - var file = processRequest.ComponentStream; - using var stream = file.Stream; - -#pragma warning disable CA5350, CA1308 // Existing SPDX document references require lowercase SHA1 checksums. - var checksum = Convert.ToHexString(await SHA1.HashDataAsync(stream, cancellationToken)).ToLowerInvariant(); -#pragma warning restore CA5350, CA1308 - stream.Seek(0, SeekOrigin.Begin); - - using var json = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); - var component = this.ReadDocument(json.RootElement, checksum, file.Location); - if (component != null) - { - processRequest.SingleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); - } - } -} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs index 15ce034a6..c53bfedb2 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs @@ -7,7 +7,6 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using System.Linq; using System.Security.Cryptography; using System.Text; -using System.Threading; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.ComponentDetection.Common.DependencyGraph; @@ -15,9 +14,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Spdx; using Microsoft.ComponentDetection.TestsUtilities; -using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; [TestClass] [TestCategory("Governance/All")] @@ -167,33 +164,6 @@ public async Task TestSbomDetector_InvalidFileAsync() components.Should().BeEmpty(); } - [TestMethod] - public async Task TestSbomDetector_LogsUnexpectedStreamErrorsAsync() - { - var error = new InvalidOperationException("Read failed."); - var stream = new Mock(); - stream.SetupGet(s => s.CanRead).Returns(true); - stream.Setup(s => s.ReadAsync(It.IsAny>(), It.IsAny())) - .ThrowsAsync(error); - var logger = new Mock>(); - - var (result, recorder) = await this.detectorTestUtility - .AddServiceMock(logger) - .WithFile("manifest.spdx.json", stream.Object) - .ExecuteDetectorAsync(); - - result.ResultCode.Should().Be(ProcessingResultCode.Success); - recorder.GetDetectedComponents().Should().BeEmpty(); - logger.Verify( - log => log.Log( - LogLevel.Error, - It.IsAny(), - It.IsAny(), - error, - It.IsAny>()), - Times.Once); - } - [TestMethod] public async Task TestSbomDetector_ExtractsCreatorToolAndOrganizationAsync() { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs index ec0c4cbab..c06cf0eb9 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs @@ -6,7 +6,6 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using System.Security.Cryptography; using System.Text; using System.Text.Json.Nodes; -using System.Threading; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.ComponentDetection.Contracts; @@ -195,21 +194,6 @@ public async Task RejectsMultipleDocumentsOrDuplicateIdentifiersAsync(int index) await this.AssertSkippedWithWarningAsync(json.ToJsonString()); } - [TestMethod] - public async Task PropagatesUnexpectedStreamErrorsAsync() - { - var stream = new Mock(); - stream.SetupGet(s => s.CanRead).Returns(true); - stream.Setup(s => s.ReadAsync(It.IsAny>(), It.IsAny())) - .ThrowsAsync(new InvalidOperationException("Read failed.")); - - Func action = () => this.DetectorTestUtility - .WithFile("manifest.spdx.json", stream.Object) - .ExecuteDetectorAsync(); - - await action.Should().ThrowAsync().WithMessage("Read failed."); - } - private async Task AssertSkippedWithWarningAsync(string content) { var logger = new Mock>(); From 1d2d6d61c8cfbf963a129ff2cd7ea04591ac58ff Mon Sep 17 00:00:00 2001 From: Sarah Oslund Date: Fri, 23 Oct 2026 12:55:35 -0700 Subject: [PATCH 6/7] fix(spdx): handle version mismatches and oversized documents Log detector version mismatches at debug level and accept stable SPDX 3.0 patch versions. Warn and skip SPDX 3.0 files that exhaust memory or overflow size limits, without stopping subsequent files. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3e4f54f6-111a-4ceb-85fb-5b08c49779af --- .../spdx/Spdx22ComponentDetector.cs | 2 +- .../spdx/Spdx30ComponentDetector.cs | 17 +++- .../SPDX22ComponentDetectorTests.cs | 33 ++++++++ .../Spdx30ComponentDetectorTests.cs | 80 +++++++++++++++++-- 4 files changed, 121 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs index 77de63730..a49bed04a 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs @@ -67,7 +67,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } else { - this.Logger.LogWarning("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", processRequest.ComponentStream.Location); + this.Logger.LogDebug("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", processRequest.ComponentStream.Location); } } catch (JsonException) diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs index c2a061a9e..1d45e0eed 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs @@ -6,6 +6,7 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; using System.Linq; using System.Security.Cryptography; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; @@ -15,6 +16,8 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; public class Spdx30ComponentDetector : FileComponentDetector, IDefaultOffComponentDetector { + private static readonly Regex SupportedVersionRegex = new Regex(@"\A3\.0\.(0|[1-9][0-9]*)\z", RegexOptions.CultureInvariant | RegexOptions.NonBacktracking); + public Spdx30ComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, @@ -63,6 +66,10 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID { this.Logger.LogError(e, "Error while reading SPDX 3.0 file at {ManifestLocation}", file.Location); } + catch (Exception e) when (e is OutOfMemoryException or OverflowException) + { + this.Logger.LogWarning(e, "SPDX 3.0 file at {ManifestLocation} exceeds available memory or supported size, skipping", file.Location); + } } private static string? ReadString(JsonElement element, string property) @@ -124,7 +131,7 @@ private static IEnumerable ReadValues(JsonElement element, string p || !root.TryGetProperty("@graph", out var graph) || graph.ValueKind != JsonValueKind.Array) { - this.Logger.LogWarning("Discovered file at {ManifestLocation} is not an SPDX 3.0 JSON-LD graph, skipping", path); + this.Logger.LogDebug("Discovered file at {ManifestLocation} is not an SPDX 3.0 JSON-LD graph, skipping", path); return null; } @@ -152,10 +159,12 @@ private static IEnumerable ReadValues(JsonElement element, string p } creationInfo = Resolve(creationInfo, elements); - var version = ReadString(creationInfo, "specVersion"); - if (version is not ("3.0.0" or "3.0.1")) + var version = ReadString(creationInfo, "specVersion") + ?? throw new JsonException("SpdxDocument creationInfo is missing specVersion."); + + if (!SupportedVersionRegex.IsMatch(version)) { - this.Logger.LogWarning("SPDX file at {ManifestLocation} has unsupported or missing specVersion {SpecVersion}, skipping", path, version); + this.Logger.LogDebug("SPDX file at {ManifestLocation} has unsupported specVersion {SpecVersion}, skipping", path, version); return null; } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs index c53bfedb2..b12c1240c 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs @@ -14,7 +14,9 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Spdx; using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; [TestClass] [TestCategory("Governance/All")] @@ -132,6 +134,37 @@ public async Task TestSbomDetector_SimpleSbomAsync() sbomComponent.CreatorOrganization.Should().Be("Microsoft"); } + [TestMethod] + [DataRow("{\"@context\":\"https://spdx.org/rdf/3.0.1/spdx-context.jsonld\",\"@graph\":[]}")] + [DataRow("{\"spdxVersion\":\"SPDX-2.3\"}")] + public async Task TestSbomDetector_OtherVersionsLogAtDebugAsync(string content) + { + var logger = new Mock>(); + var (result, recorder) = await this.detectorTestUtility + .AddServiceMock(logger) + .WithFile("manifest.spdx.json", content) + .ExecuteDetectorAsync(); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + recorder.GetDetectedComponents().Should().BeEmpty(); + logger.Verify( + log => log.Log( + LogLevel.Debug, + It.IsAny(), + It.Is((value, _) => value.ToString().Contains("not SPDX-2.2")), + It.IsAny(), + It.IsAny>()), + Times.Once); + logger.Verify( + log => log.Log( + It.Is(level => level >= LogLevel.Warning), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>()), + Times.Never); + } + [TestMethod] public async Task TestSbomDetector_BlankJsonAsync() { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs index c06cf0eb9..b44d81724 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs @@ -6,6 +6,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using System.Security.Cryptography; using System.Text; using System.Text.Json.Nodes; +using System.Threading; using System.Threading.Tasks; using AwesomeAssertions; using Microsoft.ComponentDetection.Contracts; @@ -67,6 +68,8 @@ public class Spdx30ComponentDetectorTests : BaseDetectorTest>(); + var stream = new Mock(); + stream.SetupGet(value => value.CanRead).Returns(true); + stream.Setup(value => value.ReadAsync(It.IsAny>(), It.IsAny())) + .Throws(exception); + + var (result, recorder) = await this.DetectorTestUtility + .AddServiceMock(logger) + .WithFile("large.spdx.json", stream.Object) + .WithFile("manifest.spdx.json", Manifest) + .ExecuteDetectorAsync(); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + recorder.GetDetectedComponents().Should().ContainSingle(); + logger.Verify( + log => log.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((value, _) => value.ToString()!.Contains("large.spdx.json", StringComparison.Ordinal)), + exception, + It.IsAny>()), + Times.Once); } [TestMethod] @@ -194,7 +248,9 @@ public async Task RejectsMultipleDocumentsOrDuplicateIdentifiersAsync(int index) await this.AssertSkippedWithWarningAsync(json.ToJsonString()); } - private async Task AssertSkippedWithWarningAsync(string content) + private Task AssertSkippedWithWarningAsync(string content) => this.AssertSkippedAsync(content, LogLevel.Warning); + + private async Task AssertSkippedAsync(string content, LogLevel expectedLevel) { var logger = new Mock>(); var (result, recorder) = await this.DetectorTestUtility @@ -206,11 +262,23 @@ private async Task AssertSkippedWithWarningAsync(string content) recorder.GetDetectedComponents().Should().BeEmpty(); logger.Verify( log => log.Log( - LogLevel.Warning, + expectedLevel, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), Times.AtLeastOnce); + + if (expectedLevel == LogLevel.Debug) + { + logger.Verify( + log => log.Log( + It.Is(level => level >= LogLevel.Warning), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>()), + Times.Never); + } } } From a7e6eb1f9ae9909c3cac229120720005db2e641d Mon Sep 17 00:00:00 2001 From: Sarah Oslund Date: Fri, 23 Oct 2026 14:08:06 -0700 Subject: [PATCH 7/7] refactor(spdx): replace version regex with explicit validation Preserve stable 3.0.x matching, including multi-digit patches, while rejecting leading zeros and non-ASCII digits. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3e4f54f6-111a-4ceb-85fb-5b08c49779af --- .../spdx/Spdx30ComponentDetector.cs | 18 ++++++++++++++---- .../Spdx30ComponentDetectorTests.cs | 2 ++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs index 1d45e0eed..384e3ac80 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx30ComponentDetector.cs @@ -6,7 +6,6 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; using System.Linq; using System.Security.Cryptography; using System.Text.Json; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; @@ -16,8 +15,6 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; public class Spdx30ComponentDetector : FileComponentDetector, IDefaultOffComponentDetector { - private static readonly Regex SupportedVersionRegex = new Regex(@"\A3\.0\.(0|[1-9][0-9]*)\z", RegexOptions.CultureInvariant | RegexOptions.NonBacktracking); - public Spdx30ComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, @@ -72,6 +69,19 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } } + private static bool IsSupportedVersion(string version) + { + const string prefix = "3.0."; + if (!version.StartsWith(prefix, StringComparison.Ordinal) || version.Length == prefix.Length) + { + return false; + } + + var patchVersion = version[prefix.Length..]; + return (patchVersion.Length == 1 || patchVersion[0] != '0') + && patchVersion.All(character => character is >= '0' and <= '9'); + } + private static string? ReadString(JsonElement element, string property) { if (element.ValueKind != JsonValueKind.Object || !element.TryGetProperty(property, out var value)) @@ -162,7 +172,7 @@ private static IEnumerable ReadValues(JsonElement element, string p var version = ReadString(creationInfo, "specVersion") ?? throw new JsonException("SpdxDocument creationInfo is missing specVersion."); - if (!SupportedVersionRegex.IsMatch(version)) + if (!IsSupportedVersion(version)) { this.Logger.LogDebug("SPDX file at {ManifestLocation} has unsupported specVersion {SpecVersion}, skipping", path, version); return null; diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs index b44d81724..07358cbd2 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Spdx30ComponentDetectorTests.cs @@ -194,6 +194,8 @@ public async Task SkipsInvalidDocumentsWithWarningAsync(int index, string proper [DataRow(1, "specVersion", "\"3.1.0\"")] [DataRow(1, "specVersion", "\"3.0.1-draft\"")] [DataRow(1, "specVersion", "\"3.0\"")] + [DataRow(1, "specVersion", "\"3.0.\"")] + [DataRow(1, "specVersion", "\"3.0.\\u0661\"")] [DataRow(1, "specVersion", "\"3.0.01\"")] [DataRow(1, "specVersion", "\"3.0.-1\"")] [DataRow(1, "specVersion", "\"3.0.1.0\"")]