gitea-mirror/trivy
1
0
Fork 0
mirror of https://github.com/aquasecurity/trivy.git synced 2025-12-12 07:40:48 -08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(misconf): Simplify misconfig checks bundle parsing (#8533)

Browse Source
This commit is contained in:
simar7
2025-03-21 16:38:26 -06:00
committed by GitHub
parent 8e1019d82c
commit 68b164ddf4
7 changed files with 245 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
pkg/commands/artifact/run.go
View File

@@ -25,6 +25,7 @@ import (
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/misconf"
"github.com/aquasecurity/trivy/pkg/module"
"github.com/aquasecurity/trivy/pkg/policy"
pkgReport "github.com/aquasecurity/trivy/pkg/report"
"github.com/aquasecurity/trivy/pkg/result"
"github.com/aquasecurity/trivy/pkg/rpc/client"
@@ -635,19 +636,27 @@ func initMisconfScannerOption(ctx context.Context, opts flag.Options) (misconf.S
ctx = log.WithContextPrefix(ctx, log.PrefixMisconfiguration)
log.InfoContext(ctx, "Misconfiguration scanning is enabled")
var downloadedPolicyPaths []string
var downloadedPolicyPath string
var disableEmbedded bool
downloadedPolicyPaths, err := operation.InitBuiltinChecks(ctx, opts.CacheDir, opts.Quiet, opts.SkipCheckUpdate, opts.MisconfOptions.ChecksBundleRepository, opts.RegistryOpts())
c, err := policy.NewClient(opts.CacheDir, opts.Quiet, opts.MisconfOptions.ChecksBundleRepository)
if err != nil {
if !opts.SkipCheckUpdate {
log.ErrorContext(ctx, "Falling back to embedded checks", log.Err(err))
}
return misconf.ScannerOption{}, xerrors.Errorf("check client error: %w", err)
}
downloadedPolicyPath, err = operation.InitBuiltinChecks(ctx, c, opts.SkipCheckUpdate, opts.RegistryOpts())
if err != nil {
log.ErrorContext(ctx, "Falling back to embedded checks", log.Err(err))
} else {
log.DebugContext(ctx, "Checks successfully loaded from disk")
disableEmbedded = true
}
policyPaths := slices.Clone(opts.CheckPaths)
if downloadedPolicyPath != "" {
policyPaths = append(policyPaths, downloadedPolicyPath)
}
configSchemas, err := misconf.LoadConfigSchemas(opts.ConfigFileSchemas)
if err != nil {
return misconf.ScannerOption{}, xerrors.Errorf("load schemas error: %w", err)
@@ -656,7 +665,7 @@ func initMisconfScannerOption(ctx context.Context, opts flag.Options) (misconf.S
return misconf.ScannerOption{
Trace: opts.Trace,
Namespaces: append(opts.CheckNamespaces, rego.BuiltinNamespaces()...),
PolicyPaths: append(opts.CheckPaths, downloadedPolicyPaths...),
PolicyPaths: policyPaths,
DataPaths: opts.DataPaths,
HelmValues: opts.HelmValues,
HelmValueFiles: opts.HelmValueFiles,

42
pkg/commands/operation/operation.go
View File

@@ -12,6 +12,7 @@ import (
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/flag"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/misconf"
"github.com/aquasecurity/trivy/pkg/policy"
"github.com/aquasecurity/trivy/pkg/types"
"github.com/aquasecurity/trivy/pkg/vex"
@@ -78,41 +79,36 @@ func DownloadVEXRepositories(ctx context.Context, opts flag.Options) error {
}
// InitBuiltinChecks downloads the built-in policies and loads them
func InitBuiltinChecks(ctx context.Context, cacheDir string, quiet, skipUpdate bool, checkBundleRepository string, registryOpts ftypes.RegistryOptions) ([]string, error) {
func InitBuiltinChecks(ctx context.Context, client *policy.Client, skipUpdate bool, registryOpts ftypes.RegistryOptions) (string, error) {
mu.Lock()
defer mu.Unlock()
var err error
client, err := policy.NewClient(cacheDir, quiet, checkBundleRepository)
if err != nil {
return nil, xerrors.Errorf("check client error: %w", err)
if skipUpdate {
log.Info("No downloadable checks were loaded as --skip-check-update is enabled, loading from existing cache...")
path := client.LoadBuiltinChecks()
_, _, err := misconf.CheckPathExists(path)
if err != nil {
return "", xerrors.Errorf("Failed to load existing cache, err: %s", err.Error())
}
return path, nil
}
needsUpdate := false
if !skipUpdate {
needsUpdate, err = client.NeedsUpdate(ctx, registryOpts)
if err != nil {
return nil, xerrors.Errorf("unable to check if built-in policies need to be updated: %w", err)
}
needsUpdate, err := client.NeedsUpdate(ctx, registryOpts)
if err != nil {
return "", xerrors.Errorf("unable to check if built-in policies need to be updated: %w", err)
}
if needsUpdate {
log.InfoContext(ctx, "Need to update the built-in checks")
log.InfoContext(ctx, "Downloading the built-in checks...")
log.InfoContext(ctx, "Need to update the checks bundle")
log.InfoContext(ctx, "Downloading the checks bundle...")
if err = client.DownloadBuiltinChecks(ctx, registryOpts); err != nil {
return nil, xerrors.Errorf("failed to download built-in policies: %w", err)
return "", xerrors.Errorf("failed to download checks bundle: %w", err)
}
}
policyPaths, err := client.LoadBuiltinChecks()
if err != nil {
if skipUpdate {
msg := "No downloadable policies were loaded as --skip-check-update is enabled"
log.Info(msg)
return nil, xerrors.Errorf(msg)
}
return nil, xerrors.Errorf("check load error: %w", err)
}
return policyPaths, nil
return client.LoadBuiltinChecks(), nil
}
func Exit(opts flag.Options, failedResults bool, m types.Metadata) error {

183
pkg/commands/operation/operation_test.go Normal file
View File

@@ -0,0 +1,183 @@
package operation
import (
"bytes"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"testing"
"time"
v1 "github.com/google/go-containerregistry/pkg/v1"
fakei "github.com/google/go-containerregistry/pkg/v1/fake"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/utils/clock"
fake "k8s.io/utils/clock/testing"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/oci"
"github.com/aquasecurity/trivy/pkg/policy"
)
type stubLayer struct {
v1.Layer
}
func (f stubLayer) MediaType() (types.MediaType, error) {
return "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip", nil
}
func newStubLayer(t *testing.T) v1.Layer {
layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewBufferString("foo bar baz")), nil
})
require.NoError(t, err)
require.NotNil(t, layer)
return stubLayer{layer}
}
func TestInitBuiltinChecks(t *testing.T) {
type digestReturns struct {
h v1.Hash
err error
}
type layersReturns struct {
layers []v1.Layer
}
tests := []struct {
name string
metadata any
checkDir string
skipUpdate bool
wantErr string
layersReturns layersReturns
digestReturns digestReturns
want *policy.Metadata
clock clock.Clock
}{
{
name: "happy path - no need to update",
digestReturns: digestReturns{
h: v1.Hash{
Algorithm: "sha256",
Hex: "01e033e78bd8a59fa4f4577215e7da06c05e1152526094d8d79d2aa06e98cb9d",
},
},
checkDir: "policy",
clock: fake.NewFakeClock(time.Date(1992, 1, 1, 1, 0, 0, 0, time.UTC)),
metadata: policy.Metadata{
Digest: `sha256:922e50f14ab484f11ae65540c3d2d76009020213f1027d4331d31141575e5414`,
DownloadedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
},
skipUpdate: false,
},
{
name: "skip update flag set with no existing cache to fallback to",
skipUpdate: true,
checkDir: "policy",
wantErr: "Failed to load existing cache",
},
{
name: "skip update flag set with existing cache to fallback to",
skipUpdate: true,
checkDir: filepath.Join("policy", "content"),
clock: fake.NewFakeClock(time.Date(1992, 1, 1, 1, 0, 0, 0, time.UTC)),
metadata: policy.Metadata{
Digest: `sha256:922e50f14ab484f11ae65540c3d2d76009020213f1027d4331d31141575e5414`,
DownloadedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
},
},
{
name: "needs update returns an error",
digestReturns: digestReturns{
err: errors.New("digest error"),
},
metadata: policy.Metadata{
Digest: `sha256:922e50f14ab484f11ae65540c3d2d76009020213f1027d4331d31141575e5414`,
DownloadedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
},
checkDir: "policy",
clock: fake.NewFakeClock(time.Date(3000, 1, 1, 1, 0, 0, 0, time.UTC)),
wantErr: "unable to check if built-in policies need to be updated",
},
{
name: "sad: download builtin checks returns an error",
clock: fake.NewFakeClock(time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC)),
layersReturns: layersReturns{
layers: []v1.Layer{newStubLayer(t)},
},
digestReturns: digestReturns{
err: errors.New("error"),
},
want: &policy.Metadata{
Digest: "sha256:01e033e78bd8a59fa4f4577215e7da06c05e1152526094d8d79d2aa06e98cb9d",
DownloadedAt: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
wantErr: "failed to download checks bundle",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := t.Context()
// Set up a temporary directory
tmpDir := t.TempDir()
// Create a check directory
err := os.MkdirAll(filepath.Join(tmpDir, tt.checkDir), os.ModePerm)
require.NoError(t, err)
if tt.metadata != nil {
b, err := json.Marshal(tt.metadata)
require.NoError(t, err)
// Write a metadata file
metadataPath := filepath.Join(tmpDir, "policy", "metadata.json")
err = os.WriteFile(metadataPath, b, os.ModePerm)
require.NoError(t, err)
}
// Mock image
img := new(fakei.FakeImage)
img.LayersReturns([]v1.Layer{newStubLayer(t)}, nil)
img.DigestReturns(tt.digestReturns.h, tt.digestReturns.err)
img.ManifestReturns(&v1.Manifest{
Layers: []v1.Descriptor{
{
MediaType: "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip",
Size: 100,
Digest: v1.Hash{
Algorithm: "sha256",
Hex: "cba33656188782852f58993f45b68bfb8577f64cdcf02a604e3fc2afbeb5f2d8",
},
Annotations: map[string]string{
"org.opencontainers.image.title": "bundle.tar.gz",
},
},
},
}, nil)
// Mock OCI artifact
art := oci.NewArtifact("repo", ftypes.RegistryOptions{}, oci.WithImage(img))
c, err := policy.NewClient(tmpDir, true, "", policy.WithOCIArtifact(art), policy.WithClock(tt.clock))
require.NoError(t, err)
got, err := InitBuiltinChecks(ctx, c, tt.skipUpdate, ftypes.RegistryOptions{})
if tt.wantErr != "" {
require.ErrorContains(t, err, tt.wantErr)
assert.Empty(t, got)
return
}
require.NoError(t, err)
assert.NotEmpty(t, got)
})
}
}

7
pkg/k8s/commands/cluster.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/aquasecurity/trivy/pkg/commands/operation"
"github.com/aquasecurity/trivy/pkg/flag"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/policy"
"github.com/aquasecurity/trivy/pkg/types"
)
@@ -68,8 +69,8 @@ func nodeCollectorOptions(ctx context.Context, opts flag.Options) []trivyk8s.Nod
}
ctx = log.WithContextPrefix(ctx, log.PrefixMisconfiguration)
contentPath, err := operation.InitBuiltinChecks(ctx, opts.CacheDir, opts.Quiet, opts.SkipCheckUpdate,
opts.MisconfOptions.ChecksBundleRepository, opts.RegistryOpts())
c, _ := policy.NewClient(opts.CacheDir, opts.Quiet, opts.MisconfOptions.ChecksBundleRepository)
contentPath, err := operation.InitBuiltinChecks(ctx, c, opts.SkipCheckUpdate, opts.RegistryOpts())
if err != nil {
log.Error("Falling back to embedded checks", log.Err(err))
nodeCollectorOptions = append(nodeCollectorOptions,
@@ -81,7 +82,7 @@ func nodeCollectorOptions(ctx context.Context, opts flag.Options) []trivyk8s.Nod
complianceCommandsIDs := getComplianceCommands(opts)
nodeCollectorOptions = append(nodeCollectorOptions, []trivyk8s.NodeCollectorOption{
trivyk8s.WithCommandPaths(contentPath),
trivyk8s.WithCommandPaths([]string{contentPath}),
trivyk8s.WithSpecCommandIds(complianceCommandsIDs),
}...)
return nodeCollectorOptions

24
pkg/misconf/scanner.go
View File

@@ -369,6 +369,20 @@ func createConfigFS(paths []string) (fs.FS, error) {
return mfs, nil
}
func CheckPathExists(path string) (fs.FileInfo, string, error) {
abs, err := filepath.Abs(path)
if err != nil {
return nil, "", xerrors.Errorf("failed to derive absolute path from '%s': %w", path, err)
}
fi, err := os.Stat(abs)
if errors.Is(err, os.ErrNotExist) {
return nil, "", xerrors.Errorf("check file %q not found", abs)
} else if err != nil {
return nil, "", xerrors.Errorf("file %q stat error: %w", abs, err)
}
return fi, abs, nil
}
func CreatePolicyFS(policyPaths []string) (fs.FS, []string, error) {
if len(policyPaths) == 0 {
return nil, nil, nil
@@ -376,15 +390,9 @@ func CreatePolicyFS(policyPaths []string) (fs.FS, []string, error) {
mfs := mapfs.New()
for _, p := range policyPaths {
abs, err := filepath.Abs(p)
fi, abs, err := CheckPathExists(p)
if err != nil {
return nil, nil, xerrors.Errorf("failed to derive absolute path from '%s': %w", p, err)
}
fi, err := os.Stat(abs)
if errors.Is(err, os.ErrNotExist) {
return nil, nil, xerrors.Errorf("policy file %q not found", abs)
} else if err != nil {
return nil, nil, xerrors.Errorf("file %q stat error: %w", abs, err)
return nil, nil, err
}
if fi.IsDir() {

33
pkg/policy/policy.go
View File

@@ -8,7 +8,6 @@ import (
"path/filepath"
"time"
"github.com/open-policy-agent/opa/v1/bundle"
"golang.org/x/xerrors"
"k8s.io/utils/clock"
@@ -124,30 +123,8 @@ func (c *Client) DownloadBuiltinChecks(ctx context.Context, registryOpts types.R
}
// LoadBuiltinChecks loads default policies
func (c *Client) LoadBuiltinChecks() ([]string, error) {
f, err := os.Open(c.manifestPath())
if err != nil {
return nil, xerrors.Errorf("manifest file open error (%s): %w", c.manifestPath(), err)
}
defer f.Close()
var manifest bundle.Manifest
if err = json.NewDecoder(f).Decode(&manifest); err != nil {
return nil, xerrors.Errorf("json decode error (%s): %w", c.manifestPath(), err)
}
// If the "roots" field is not included in the manifest it defaults to [""]
// which means that ALL data and check must come from the bundle.
if manifest.Roots == nil || len(*manifest.Roots) == 0 {
return []string{c.contentDir()}, nil
}
var policyPaths []string
for _, root := range *manifest.Roots {
policyPaths = append(policyPaths, filepath.Join(c.contentDir(), root))
}
return policyPaths, nil
func (c *Client) LoadBuiltinChecks() string {
return c.contentDir()
}
// NeedsUpdate returns if the default check should be updated
@@ -190,14 +167,10 @@ func (c *Client) metadataPath() string {
return filepath.Join(c.policyDir, "metadata.json")
}
func (c *Client) manifestPath() string {
return filepath.Join(c.contentDir(), bundle.ManifestExt)
}
func (c *Client) updateMetadata(digest string, now time.Time) error {
f, err := os.Create(c.metadataPath())
if err != nil {
return xerrors.Errorf("failed to open a check manifest: %w", err)
return xerrors.Errorf("failed to open checks bundle metadata: %w", err)
}
defer f.Close()

32
pkg/policy/policy_test.go
View File

@@ -58,39 +58,17 @@ func newBrokenLayer(t *testing.T) v1.Layer {
return brokenLayer{layer}
}
func TestClient_LoadBuiltinPolicies(t *testing.T) {
func TestClient_LoadBuiltinChecks(t *testing.T) {
tests := []struct {
name string
cacheDir string
want []string
want string
wantErr string
}{
{
name: "happy path",
cacheDir: "testdata/happy",
want: []string{
filepath.Join("testdata", "happy", "policy", "content", "kubernetes"),
filepath.Join("testdata", "happy", "policy", "content", "docker"),
},
},
{
name: "empty roots",
cacheDir: "testdata/empty",
want: []string{
filepath.Join("testdata", "empty", "policy", "content"),
},
},
{
name: "broken manifest",
cacheDir: "testdata/broken",
want: []string{},
wantErr: "json decode error",
},
{
name: "no such file",
cacheDir: "testdata/unknown",
want: []string{},
wantErr: "manifest file open error",
want: filepath.Join("testdata", "happy", "policy", "content"),
},
}
for _, tt := range tests {
@@ -119,7 +97,7 @@ func TestClient_LoadBuiltinPolicies(t *testing.T) {
c, err := policy.NewClient(tt.cacheDir, true, "", policy.WithOCIArtifact(art))
require.NoError(t, err)
got, err := c.LoadBuiltinChecks()
got := c.LoadBuiltinChecks()
if tt.wantErr != "" {
require.ErrorContains(t, err, tt.wantErr)
return
@@ -265,7 +243,7 @@ func TestClient_NeedsUpdate(t *testing.T) {
}
}
func TestClient_DownloadBuiltinPolicies(t *testing.T) {
func TestClient_DownloadBuiltinChecks(t *testing.T) {
type digestReturns struct {
h v1.Hash
err error