diff --git a/.github/workflows/spdx-cron.yaml b/.github/workflows/spdx-cron.yaml index 1218468f5e..82333b151e 100644 --- a/.github/workflows/spdx-cron.yaml +++ b/.github/workflows/spdx-cron.yaml @@ -20,12 +20,12 @@ jobs: - name: Install Go tools run: go install tool # GOBIN is added to the PATH by the setup-go action - - name: Check if SPDX exceptions are up-to-date + - name: Check if SPDX license IDs and exceptions are up-to-date id: exceptions_check run: | - mage spdx:updateLicenseExceptions + mage spdx:updateLicenseEntries if [ -n "$(git status --porcelain)" ]; then - echo "Run 'mage spdx:updateLicenseExceptions' and push it" + echo "Run 'mage spdx:updateLicenseEntries' and push it" echo "send_notify=true" >> $GITHUB_OUTPUT fi diff --git a/magefiles/magefile.go b/magefiles/magefile.go index 15f0a481c8..1ad8644239 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -506,7 +506,7 @@ func (Helm) UpdateVersion() error { type SPDX mg.Namespace -// UpdateLicenseExceptions updates 'exception.json' with SPDX license exceptions -func (SPDX) UpdateLicenseExceptions() error { +// UpdateLicenseEntries updates both SPDX license IDs and exceptions +func (SPDX) UpdateLicenseEntries() error { return sh.RunWith(ENV, "go", "run", "-tags=mage_spdx", "./magefiles/spdx.go") } diff --git a/magefiles/spdx.go b/magefiles/spdx.go index 258afcd0b1..6b6da85a5e 100644 --- a/magefiles/spdx.go +++ b/magefiles/spdx.go @@ -17,11 +17,22 @@ import ( ) const ( + expressionDir = "./pkg/licensing/expression" + exceptionFileName = "exceptions.json" - exceptionDir = "./pkg/licensing/expression" exceptionURL = "https://spdx.org/licenses/exceptions.json" + + licenseFileName = "licenses.json" + licenseURL = "https://spdx.org/licenses/licenses.json" ) +func main() { + if err := run(); err != nil { + log.Fatal("Fatal error", log.Err(err)) + } + +} + type Exceptions struct { Exceptions []Exception `json:"exceptions"` } @@ -30,49 +41,76 @@ type Exception struct { ID string `json:"licenseExceptionId"` } -func main() { - if err := run(); err != nil { - log.Fatal("Fatal error", log.Err(err)) +// run downloads SPDX licenses and exceptions, extracts only IDs and writes flat arrays into the `expression` package. +func run() error { + if err := updateLicenses(); err != nil { + return err } - + return updateExceptions() } -// run downloads exceptions.json file, takes only IDs and saves into `expression` package. -func run() error { - tmpDir, err := downloader.DownloadToTempDir(context.Background(), exceptionURL, downloader.Options{}) - if err != nil { - return xerrors.Errorf("unable to download exceptions.json file: %w", err) - } - tmpFile, err := os.ReadFile(filepath.Join(tmpDir, exceptionFileName)) - if err != nil { - return xerrors.Errorf("unable to read exceptions.json file: %w", err) - } - - exceptions := Exceptions{} - if err = json.Unmarshal(tmpFile, &exceptions); err != nil { - return xerrors.Errorf("unable to unmarshal exceptions.json file: %w", err) - } - - exs := lo.Map(exceptions.Exceptions, func(ex Exception, _ int) string { - return ex.ID +func updateExceptions() error { + return fetchAndWrite(exceptionURL, exceptionFileName, filepath.Join(expressionDir, exceptionFileName), func(b []byte) ([]string, error) { + var exceptions Exceptions + if err := json.Unmarshal(b, &exceptions); err != nil { + return nil, xerrors.Errorf("unable to unmarshal exceptions.json file: %w", err) + } + exs := lo.Map(exceptions.Exceptions, func(ex Exception, _ int) string { return ex.ID }) + return exs, nil }) - sort.Strings(exs) +} - exceptionFile := filepath.Join(exceptionDir, exceptionFileName) - f, err := os.Create(exceptionFile) +type Licenses struct { + Licenses []License `json:"licenses"` +} + +type License struct { + ID string `json:"licenseId"` +} + +func updateLicenses() error { + return fetchAndWrite(licenseURL, licenseFileName, filepath.Join(expressionDir, licenseFileName), func(b []byte) ([]string, error) { + var licenses Licenses + if err := json.Unmarshal(b, &licenses); err != nil { + return nil, xerrors.Errorf("unable to unmarshal licenses.json file: %w", err) + } + ids := lo.Map(licenses.Licenses, func(l License, _ int) string { return l.ID }) + return ids, nil + }) +} + +// fetchAndWrite downloads a SPDX index file, extracts IDs using extractor, sorts and writes them to destPath +func fetchAndWrite(url, tmpFileName, destPath string, extractor func([]byte) ([]string, error)) error { + tmpDir, err := downloader.DownloadToTempDir(context.Background(), url, downloader.Options{}) if err != nil { - return xerrors.Errorf("unable to create file %s: %w", exceptionFile, err) + return xerrors.Errorf("unable to download %s: %w", tmpFileName, err) + } + tmpFile, err := os.ReadFile(filepath.Join(tmpDir, tmpFileName)) + if err != nil { + return xerrors.Errorf("unable to read %s: %w", tmpFileName, err) + } + + ids, err := extractor(tmpFile) + if err != nil { + return err + } + sort.Strings(ids) + return writeIDs(destPath, ids) +} + +func writeIDs(path string, ids []string) error { + f, err := os.Create(path) + if err != nil { + return xerrors.Errorf("unable to create file %s: %w", path, err) } defer f.Close() - e, err := json.Marshal(exs) + b, err := json.Marshal(ids) if err != nil { - return xerrors.Errorf("unable to marshal exceptions list: %w", err) + return xerrors.Errorf("unable to marshal id list: %w", err) } - - if _, err = f.Write(e); err != nil { - return xerrors.Errorf("unable to write exceptions list: %w", err) + if _, err = f.Write(b); err != nil { + return xerrors.Errorf("unable to write id list: %w", err) } - return nil } diff --git a/pkg/licensing/expression/category.go b/pkg/licensing/expression/category.go index d9655516b3..ce520159c6 100644 --- a/pkg/licensing/expression/category.go +++ b/pkg/licensing/expression/category.go @@ -375,34 +375,24 @@ var ( var spdxLicenses = set.New[string]() +//go:embed licenses.json +var licenses []byte + var initSpdxLicenses = sync.OnceFunc(func() { if spdxLicenses.Size() > 0 { return } - licenseSlices := [][]string{ - ForbiddenLicenses, - RestrictedLicenses, - ReciprocalLicenses, - NoticeLicenses, - PermissiveLicenses, - UnencumberedLicenses, + var lics []string + if err := json.Unmarshal(licenses, &lics); err != nil { + log.WithPrefix(log.PrefixSPDX).Warn("Unable to parse SPDX license file", log.Err(err)) + return } - for _, licenseSlice := range licenseSlices { - spdxLicenses.Append(licenseSlice...) - } - - // Save GNU licenses with "-or-later" and `"-only" suffixes - for _, l := range GnuLicenses { - license := SimpleExpr{ - License: l, - } - spdxLicenses.Append(license.String()) - - license.HasPlus = true - spdxLicenses.Append(license.String()) - } + // SPDX license list is case-insensitive. Store in upper case for simplicity. + spdxLicenses.Append(lo.Map(lics, func(l string, _ int) string { + return strings.ToUpper(l) + })...) }) //go:embed exceptions.json @@ -429,7 +419,7 @@ var initSpdxExceptions = sync.OnceFunc(func() { func ValidateSPDXLicense(license string) bool { initSpdxLicenses() - return spdxLicenses.Contains(license) + return spdxLicenses.Contains(strings.ToUpper(license)) } // ValidateSPDXException returns true if SPDX exception list contain exceptionID diff --git a/pkg/licensing/expression/licenses.json b/pkg/licensing/expression/licenses.json new file mode 100644 index 0000000000..ad139ecac4 --- /dev/null +++ b/pkg/licensing/expression/licenses.json @@ -0,0 +1 @@ +["0BSD","3D-Slicer-1.0","AAL","ADSL","AFL-1.1","AFL-1.2","AFL-2.0","AFL-2.1","AFL-3.0","AGPL-1.0","AGPL-1.0-only","AGPL-1.0-or-later","AGPL-3.0","AGPL-3.0-only","AGPL-3.0-or-later","AMD-newlib","AMDPLPA","AML","AML-glslang","AMPAS","ANTLR-PD","ANTLR-PD-fallback","APAFML","APL-1.0","APSL-1.0","APSL-1.1","APSL-1.2","APSL-2.0","ASWF-Digital-Assets-1.0","ASWF-Digital-Assets-1.1","Abstyles","AdaCore-doc","Adobe-2006","Adobe-Display-PostScript","Adobe-Glyph","Adobe-Utopia","Afmparse","Aladdin","Apache-1.0","Apache-1.1","Apache-2.0","App-s2p","Arphic-1999","Artistic-1.0","Artistic-1.0-Perl","Artistic-1.0-cl8","Artistic-2.0","Artistic-dist","Aspell-RU","BSD-1-Clause","BSD-2-Clause","BSD-2-Clause-Darwin","BSD-2-Clause-FreeBSD","BSD-2-Clause-NetBSD","BSD-2-Clause-Patent","BSD-2-Clause-Views","BSD-2-Clause-first-lines","BSD-2-Clause-pkgconf-disclaimer","BSD-3-Clause","BSD-3-Clause-Attribution","BSD-3-Clause-Clear","BSD-3-Clause-HP","BSD-3-Clause-LBNL","BSD-3-Clause-Modification","BSD-3-Clause-No-Military-License","BSD-3-Clause-No-Nuclear-License","BSD-3-Clause-No-Nuclear-License-2014","BSD-3-Clause-No-Nuclear-Warranty","BSD-3-Clause-Open-MPI","BSD-3-Clause-Sun","BSD-3-Clause-acpica","BSD-3-Clause-flex","BSD-4-Clause","BSD-4-Clause-Shortened","BSD-4-Clause-UC","BSD-4.3RENO","BSD-4.3TAHOE","BSD-Advertising-Acknowledgement","BSD-Attribution-HPND-disclaimer","BSD-Inferno-Nettverk","BSD-Protection","BSD-Source-Code","BSD-Source-beginning-file","BSD-Systemics","BSD-Systemics-W3Works","BSL-1.0","BUSL-1.1","Baekmuk","Bahyph","Barr","Beerware","BitTorrent-1.0","BitTorrent-1.1","Bitstream-Charter","Bitstream-Vera","BlueOak-1.0.0","Boehm-GC","Boehm-GC-without-fee","Borceux","Brian-Gladman-2-Clause","Brian-Gladman-3-Clause","C-UDA-1.0","CAL-1.0","CAL-1.0-Combined-Work-Exception","CATOSL-1.1","CC-BY-1.0","CC-BY-2.0","CC-BY-2.5","CC-BY-2.5-AU","CC-BY-3.0","CC-BY-3.0-AT","CC-BY-3.0-AU","CC-BY-3.0-DE","CC-BY-3.0-IGO","CC-BY-3.0-NL","CC-BY-3.0-US","CC-BY-4.0","CC-BY-NC-1.0","CC-BY-NC-2.0","CC-BY-NC-2.5","CC-BY-NC-3.0","CC-BY-NC-3.0-DE","CC-BY-NC-4.0","CC-BY-NC-ND-1.0","CC-BY-NC-ND-2.0","CC-BY-NC-ND-2.5","CC-BY-NC-ND-3.0","CC-BY-NC-ND-3.0-DE","CC-BY-NC-ND-3.0-IGO","CC-BY-NC-ND-4.0","CC-BY-NC-SA-1.0","CC-BY-NC-SA-2.0","CC-BY-NC-SA-2.0-DE","CC-BY-NC-SA-2.0-FR","CC-BY-NC-SA-2.0-UK","CC-BY-NC-SA-2.5","CC-BY-NC-SA-3.0","CC-BY-NC-SA-3.0-DE","CC-BY-NC-SA-3.0-IGO","CC-BY-NC-SA-4.0","CC-BY-ND-1.0","CC-BY-ND-2.0","CC-BY-ND-2.5","CC-BY-ND-3.0","CC-BY-ND-3.0-DE","CC-BY-ND-4.0","CC-BY-SA-1.0","CC-BY-SA-2.0","CC-BY-SA-2.0-UK","CC-BY-SA-2.1-JP","CC-BY-SA-2.5","CC-BY-SA-3.0","CC-BY-SA-3.0-AT","CC-BY-SA-3.0-DE","CC-BY-SA-3.0-IGO","CC-BY-SA-4.0","CC-PDDC","CC-PDM-1.0","CC-SA-1.0","CC0-1.0","CDDL-1.0","CDDL-1.1","CDL-1.0","CDLA-Permissive-1.0","CDLA-Permissive-2.0","CDLA-Sharing-1.0","CECILL-1.0","CECILL-1.1","CECILL-2.0","CECILL-2.1","CECILL-B","CECILL-C","CERN-OHL-1.1","CERN-OHL-1.2","CERN-OHL-P-2.0","CERN-OHL-S-2.0","CERN-OHL-W-2.0","CFITSIO","CMU-Mach","CMU-Mach-nodoc","CNRI-Jython","CNRI-Python","CNRI-Python-GPL-Compatible","COIL-1.0","CPAL-1.0","CPL-1.0","CPOL-1.02","CUA-OPL-1.0","Caldera","Caldera-no-preamble","Catharon","ClArtistic","Clips","Community-Spec-1.0","Condor-1.1","Cornell-Lossless-JPEG","Cronyx","Crossword","CryptoSwift","CrystalStacker","Cube","D-FSL-1.0","DEC-3-Clause","DL-DE-BY-2.0","DL-DE-ZERO-2.0","DOC","DRL-1.0","DRL-1.1","DSDP","DocBook-DTD","DocBook-Schema","DocBook-Stylesheet","DocBook-XML","Dotseqn","ECL-1.0","ECL-2.0","EFL-1.0","EFL-2.0","EPICS","EPL-1.0","EPL-2.0","EUDatagrid","EUPL-1.0","EUPL-1.1","EUPL-1.2","Elastic-2.0","Entessa","ErlPL-1.1","Eurosym","FBM","FDK-AAC","FSFAP","FSFAP-no-warranty-disclaimer","FSFUL","FSFULLR","FSFULLRSD","FSFULLRWD","FSL-1.1-ALv2","FSL-1.1-MIT","FTL","Fair","Ferguson-Twofish","Frameworx-1.0","FreeBSD-DOC","FreeImage","Furuseth","GCR-docs","GD","GFDL-1.1","GFDL-1.1-invariants-only","GFDL-1.1-invariants-or-later","GFDL-1.1-no-invariants-only","GFDL-1.1-no-invariants-or-later","GFDL-1.1-only","GFDL-1.1-or-later","GFDL-1.2","GFDL-1.2-invariants-only","GFDL-1.2-invariants-or-later","GFDL-1.2-no-invariants-only","GFDL-1.2-no-invariants-or-later","GFDL-1.2-only","GFDL-1.2-or-later","GFDL-1.3","GFDL-1.3-invariants-only","GFDL-1.3-invariants-or-later","GFDL-1.3-no-invariants-only","GFDL-1.3-no-invariants-or-later","GFDL-1.3-only","GFDL-1.3-or-later","GL2PS","GLWTPL","GPL-1.0","GPL-1.0+","GPL-1.0-only","GPL-1.0-or-later","GPL-2.0","GPL-2.0+","GPL-2.0-only","GPL-2.0-or-later","GPL-2.0-with-GCC-exception","GPL-2.0-with-autoconf-exception","GPL-2.0-with-bison-exception","GPL-2.0-with-classpath-exception","GPL-2.0-with-font-exception","GPL-3.0","GPL-3.0+","GPL-3.0-only","GPL-3.0-or-later","GPL-3.0-with-GCC-exception","GPL-3.0-with-autoconf-exception","Game-Programming-Gems","Giftware","Glide","Glulxe","Graphics-Gems","Gutmann","HDF5","HIDAPI","HP-1986","HP-1989","HPND","HPND-DEC","HPND-Fenneberg-Livingston","HPND-INRIA-IMAG","HPND-Intel","HPND-Kevlin-Henney","HPND-MIT-disclaimer","HPND-Markus-Kuhn","HPND-Netrek","HPND-Pbmplus","HPND-UC","HPND-UC-export-US","HPND-doc","HPND-doc-sell","HPND-export-US","HPND-export-US-acknowledgement","HPND-export-US-modify","HPND-export2-US","HPND-merchantability-variant","HPND-sell-MIT-disclaimer-xserver","HPND-sell-regexpr","HPND-sell-variant","HPND-sell-variant-MIT-disclaimer","HPND-sell-variant-MIT-disclaimer-rev","HTMLTIDY","HaskellReport","Hippocratic-2.1","IBM-pibs","ICU","IEC-Code-Components-EULA","IJG","IJG-short","IPA","IPL-1.0","ISC","ISC-Veillard","ImageMagick","Imlib2","Info-ZIP","Inner-Net-2.0","InnoSetup","Intel","Intel-ACPI","Interbase-1.0","JPL-image","JPNIC","JSON","Jam","JasPer-2.0","Kastrup","Kazlib","Knuth-CTAN","LAL-1.2","LAL-1.3","LGPL-2.0","LGPL-2.0+","LGPL-2.0-only","LGPL-2.0-or-later","LGPL-2.1","LGPL-2.1+","LGPL-2.1-only","LGPL-2.1-or-later","LGPL-3.0","LGPL-3.0+","LGPL-3.0-only","LGPL-3.0-or-later","LGPLLR","LOOP","LPD-document","LPL-1.0","LPL-1.02","LPPL-1.0","LPPL-1.1","LPPL-1.2","LPPL-1.3a","LPPL-1.3c","LZMA-SDK-9.11-to-9.20","LZMA-SDK-9.22","Latex2e","Latex2e-translated-notice","Leptonica","LiLiQ-P-1.1","LiLiQ-R-1.1","LiLiQ-Rplus-1.1","Libpng","Linux-OpenIB","Linux-man-pages-1-para","Linux-man-pages-copyleft","Linux-man-pages-copyleft-2-para","Linux-man-pages-copyleft-var","Lucida-Bitmap-Fonts","MIPS","MIT","MIT-0","MIT-CMU","MIT-Click","MIT-Festival","MIT-Khronos-old","MIT-Modern-Variant","MIT-Wu","MIT-advertising","MIT-enna","MIT-feh","MIT-open-group","MIT-testregex","MITNFA","MMIXware","MPEG-SSG","MPL-1.0","MPL-1.1","MPL-2.0","MPL-2.0-no-copyleft-exception","MS-LPL","MS-PL","MS-RL","MTLL","Mackerras-3-Clause","Mackerras-3-Clause-acknowledgment","MakeIndex","Martin-Birgmeier","McPhee-slideshow","Minpack","MirOS","Motosoto","MulanPSL-1.0","MulanPSL-2.0","Multics","Mup","NAIST-2003","NASA-1.3","NBPL-1.0","NCBI-PD","NCGL-UK-2.0","NCL","NCSA","NGPL","NICTA-1.0","NIST-PD","NIST-PD-fallback","NIST-Software","NLOD-1.0","NLOD-2.0","NLPL","NOSL","NPL-1.0","NPL-1.1","NPOSL-3.0","NRL","NTIA-PD","NTP","NTP-0","Naumen","Net-SNMP","NetCDF","Newsletr","Nokia","Noweb","Nunit","O-UDA-1.0","OAR","OCCT-PL","OCLC-2.0","ODC-By-1.0","ODbL-1.0","OFFIS","OFL-1.0","OFL-1.0-RFN","OFL-1.0-no-RFN","OFL-1.1","OFL-1.1-RFN","OFL-1.1-no-RFN","OGC-1.0","OGDL-Taiwan-1.0","OGL-Canada-2.0","OGL-UK-1.0","OGL-UK-2.0","OGL-UK-3.0","OGTSL","OLDAP-1.1","OLDAP-1.2","OLDAP-1.3","OLDAP-1.4","OLDAP-2.0","OLDAP-2.0.1","OLDAP-2.1","OLDAP-2.2","OLDAP-2.2.1","OLDAP-2.2.2","OLDAP-2.3","OLDAP-2.4","OLDAP-2.5","OLDAP-2.6","OLDAP-2.7","OLDAP-2.8","OLFL-1.3","OML","OPL-1.0","OPL-UK-3.0","OPUBL-1.0","OSET-PL-2.1","OSL-1.0","OSL-1.1","OSL-2.0","OSL-2.1","OSL-3.0","OpenPBS-2.3","OpenSSL","OpenSSL-standalone","OpenVision","PADL","PDDL-1.0","PHP-3.0","PHP-3.01","PPL","PSF-2.0","Parity-6.0.0","Parity-7.0.0","Pixar","Plexus","PolyForm-Noncommercial-1.0.0","PolyForm-Small-Business-1.0.0","PostgreSQL","Python-2.0","Python-2.0.1","QPL-1.0","QPL-1.0-INRIA-2004","Qhull","RHeCos-1.1","RPL-1.1","RPL-1.5","RPSL-1.0","RSA-MD","RSCPL","Rdisc","Ruby","Ruby-pty","SAX-PD","SAX-PD-2.0","SCEA","SGI-B-1.0","SGI-B-1.1","SGI-B-2.0","SGI-OpenGL","SGP4","SHL-0.5","SHL-0.51","SISSL","SISSL-1.2","SL","SMAIL-GPL","SMLNJ","SMPPL","SNIA","SOFA","SPL-1.0","SSH-OpenSSH","SSH-short","SSLeay-standalone","SSPL-1.0","SUL-1.0","SWL","Saxpath","SchemeReport","Sendmail","Sendmail-8.23","Sendmail-Open-Source-1.1","SimPL-2.0","Sleepycat","Soundex","Spencer-86","Spencer-94","Spencer-99","StandardML-NJ","SugarCRM-1.1.3","Sun-PPP","Sun-PPP-2000","SunPro","Symlinks","TAPR-OHL-1.0","TCL","TCP-wrappers","TGPPL-1.0","TMate","TORQUE-1.1","TOSL","TPDL","TPL-1.0","TTWL","TTYP0","TU-Berlin-1.0","TU-Berlin-2.0","TermReadKey","ThirdEye","TrustedQSL","UCAR","UCL-1.0","UMich-Merit","UPL-1.0","URT-RLE","Ubuntu-font-1.0","Unicode-3.0","Unicode-DFS-2015","Unicode-DFS-2016","Unicode-TOU","UnixCrypt","Unlicense","Unlicense-libtelnet","Unlicense-libwhirlpool","VOSTROM","VSL-1.0","Vim","W3C","W3C-19980720","W3C-20150513","WTFPL","Watcom-1.0","Widget-Workshop","Wsuipa","X11","X11-distribute-modifications-variant","X11-swapped","XFree86-1.1","XSkat","Xdebug-1.03","Xerox","Xfig","Xnet","YPL-1.0","YPL-1.1","ZPL-1.1","ZPL-2.0","ZPL-2.1","Zed","Zeeff","Zend-2.0","Zimbra-1.3","Zimbra-1.4","Zlib","any-OSI","any-OSI-perl-modules","bcrypt-Solar-Designer","blessing","bzip2-1.0.5","bzip2-1.0.6","check-cvs","checkmk","copyleft-next-0.3.0","copyleft-next-0.3.1","curl","cve-tou","diffmark","dtoa","dvipdfm","eCos-2.0","eGenix","etalab-2.0","fwlw","gSOAP-1.3b","generic-xts","gnuplot","gtkbook","hdparm","iMatix","jove","libpng-1.6.35","libpng-2.0","libselinux-1.0","libtiff","libutil-David-Nugent","lsof","magaz","mailprio","man2html","metamail","mpi-permissive","mpich2","mplus","ngrep","pkgconf","pnmstitch","psfrag","psutils","python-ldap","radvd","snprintf","softSurfer","ssh-keyscan","swrule","threeparttable","ulem","w3m","wwl","wxWindows","xinetd","xkeyboard-config-Zinoviev","xlock","xpp","xzoom","zlib-acknowledgement"] \ No newline at end of file