feat(sbom): use SPDX license IDs list to validate SPDX IDs (#9569)

.github/workflows/spdx-cron.yaml

@@ -20,12 +20,12 @@ jobs:
       - name: Install Go tools
         run: go install tool # GOBIN is added to the PATH by the setup-go action
 
-      - name: Check if SPDX exceptions are up-to-date
+      - name: Check if SPDX license IDs and exceptions are up-to-date
         id: exceptions_check
         run: |
-          mage spdx:updateLicenseExceptions
+          mage spdx:updateLicenseEntries
           if [ -n "$(git status --porcelain)" ]; then
-            echo "Run 'mage spdx:updateLicenseExceptions' and push it"
+            echo "Run 'mage spdx:updateLicenseEntries' and push it"
             echo "send_notify=true" >> $GITHUB_OUTPUT
           fi
 

magefiles/magefile.go

@@ -506,7 +506,7 @@ func (Helm) UpdateVersion() error {
 
 type SPDX mg.Namespace
 
-// UpdateLicenseExceptions updates 'exception.json' with SPDX license exceptions
-func (SPDX) UpdateLicenseExceptions() error {
+// UpdateLicenseEntries updates both SPDX license IDs and exceptions
+func (SPDX) UpdateLicenseEntries() error {
 	return sh.RunWith(ENV, "go", "run", "-tags=mage_spdx", "./magefiles/spdx.go")
 }

magefiles/spdx.go

@@ -17,11 +17,22 @@ import (
 )
 
 const (
+	expressionDir = "./pkg/licensing/expression"
+
 	exceptionFileName = "exceptions.json"
-	exceptionDir      = "./pkg/licensing/expression"
 	exceptionURL      = "https://spdx.org/licenses/exceptions.json"
+
+	licenseFileName = "licenses.json"
+	licenseURL      = "https://spdx.org/licenses/licenses.json"
 )
 
+func main() {
+	if err := run(); err != nil {
+		log.Fatal("Fatal error", log.Err(err))
+	}
+
+}
+
 type Exceptions struct {
 	Exceptions []Exception `json:"exceptions"`
 }
@@ -30,49 +41,76 @@ type Exception struct {
 	ID string `json:"licenseExceptionId"`
 }
 
-func main() {
-	if err := run(); err != nil {
-		log.Fatal("Fatal error", log.Err(err))
+// run downloads SPDX licenses and exceptions, extracts only IDs and writes flat arrays into the `expression` package.
+func run() error {
+	if err := updateLicenses(); err != nil {
+		return err
 	}
-
+	return updateExceptions()
 }
 
-// run downloads exceptions.json file, takes only IDs and saves into `expression` package.
-func run() error {
-	tmpDir, err := downloader.DownloadToTempDir(context.Background(), exceptionURL, downloader.Options{})
-	if err != nil {
-		return xerrors.Errorf("unable to download exceptions.json file: %w", err)
-	}
-	tmpFile, err := os.ReadFile(filepath.Join(tmpDir, exceptionFileName))
-	if err != nil {
-		return xerrors.Errorf("unable to read exceptions.json file: %w", err)
-	}
-
-	exceptions := Exceptions{}
-	if err = json.Unmarshal(tmpFile, &exceptions); err != nil {
-		return xerrors.Errorf("unable to unmarshal exceptions.json file: %w", err)
-	}
-
-	exs := lo.Map(exceptions.Exceptions, func(ex Exception, _ int) string {
-		return ex.ID
+func updateExceptions() error {
+	return fetchAndWrite(exceptionURL, exceptionFileName, filepath.Join(expressionDir, exceptionFileName), func(b []byte) ([]string, error) {
+		var exceptions Exceptions
+		if err := json.Unmarshal(b, &exceptions); err != nil {
+			return nil, xerrors.Errorf("unable to unmarshal exceptions.json file: %w", err)
+		}
+		exs := lo.Map(exceptions.Exceptions, func(ex Exception, _ int) string { return ex.ID })
+		return exs, nil
 	})
-	sort.Strings(exs)
+}
 
-	exceptionFile := filepath.Join(exceptionDir, exceptionFileName)
-	f, err := os.Create(exceptionFile)
+type Licenses struct {
+	Licenses []License `json:"licenses"`
+}
+
+type License struct {
+	ID string `json:"licenseId"`
+}
+
+func updateLicenses() error {
+	return fetchAndWrite(licenseURL, licenseFileName, filepath.Join(expressionDir, licenseFileName), func(b []byte) ([]string, error) {
+		var licenses Licenses
+		if err := json.Unmarshal(b, &licenses); err != nil {
+			return nil, xerrors.Errorf("unable to unmarshal licenses.json file: %w", err)
+		}
+		ids := lo.Map(licenses.Licenses, func(l License, _ int) string { return l.ID })
+		return ids, nil
+	})
+}
+
+// fetchAndWrite downloads a SPDX index file, extracts IDs using extractor, sorts and writes them to destPath
+func fetchAndWrite(url, tmpFileName, destPath string, extractor func([]byte) ([]string, error)) error {
+	tmpDir, err := downloader.DownloadToTempDir(context.Background(), url, downloader.Options{})
 	if err != nil {
-		return xerrors.Errorf("unable to create file %s: %w", exceptionFile, err)
+		return xerrors.Errorf("unable to download %s: %w", tmpFileName, err)
+	}
+	tmpFile, err := os.ReadFile(filepath.Join(tmpDir, tmpFileName))
+	if err != nil {
+		return xerrors.Errorf("unable to read %s: %w", tmpFileName, err)
+	}
+
+	ids, err := extractor(tmpFile)
+	if err != nil {
+		return err
+	}
+	sort.Strings(ids)
+	return writeIDs(destPath, ids)
+}
+
+func writeIDs(path string, ids []string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return xerrors.Errorf("unable to create file %s: %w", path, err)
 	}
 	defer f.Close()
 
-	e, err := json.Marshal(exs)
+	b, err := json.Marshal(ids)
 	if err != nil {
-		return xerrors.Errorf("unable to marshal exceptions list: %w", err)
+		return xerrors.Errorf("unable to marshal id list: %w", err)
 	}
-
-	if _, err = f.Write(e); err != nil {
-		return xerrors.Errorf("unable to write exceptions list: %w", err)
+	if _, err = f.Write(b); err != nil {
+		return xerrors.Errorf("unable to write id list: %w", err)
 	}
-
 	return nil
 }

pkg/licensing/expression/category.go

@@ -375,34 +375,24 @@ var (
 
 var spdxLicenses = set.New[string]()
 
+//go:embed licenses.json
+var licenses []byte
+
 var initSpdxLicenses = sync.OnceFunc(func() {
 	if spdxLicenses.Size() > 0 {
 		return
 	}
 
-	licenseSlices := [][]string{
-		ForbiddenLicenses,
-		RestrictedLicenses,
-		ReciprocalLicenses,
-		NoticeLicenses,
-		PermissiveLicenses,
-		UnencumberedLicenses,
+	var lics []string
+	if err := json.Unmarshal(licenses, &lics); err != nil {
+		log.WithPrefix(log.PrefixSPDX).Warn("Unable to parse SPDX license file", log.Err(err))
+		return
 	}
 
-	for _, licenseSlice := range licenseSlices {
-		spdxLicenses.Append(licenseSlice...)
-	}
-
-	// Save GNU licenses with "-or-later" and `"-only" suffixes
-	for _, l := range GnuLicenses {
-		license := SimpleExpr{
-			License: l,
-		}
-		spdxLicenses.Append(license.String())
-
-		license.HasPlus = true
-		spdxLicenses.Append(license.String())
-	}
+	// SPDX license list is case-insensitive. Store in upper case for simplicity.
+	spdxLicenses.Append(lo.Map(lics, func(l string, _ int) string {
+		return strings.ToUpper(l)
+	})...)
 })
 
 //go:embed exceptions.json
@@ -429,7 +419,7 @@ var initSpdxExceptions = sync.OnceFunc(func() {
 func ValidateSPDXLicense(license string) bool {
 	initSpdxLicenses()
 
-	return spdxLicenses.Contains(license)
+	return spdxLicenses.Contains(strings.ToUpper(license))
 }
 
 // ValidateSPDXException returns true if SPDX exception list contain exceptionID