chore(dep): update (#357)

* feat(client): retry HTTP request when getting an unavailable error

* fix(integration-test): use a snapshot database for Docker mode (#352)

* fix(integration): add a binary name

The first argument is used for the program name. --skip-update was
ignored.

* fix(integration): use a snapshot database

After a new vulnerability is found, this test fails

* chore(integration): add t.Run

* refactor(client): functionalize common processes

* refactor(client): remove unused const

go.mod

@@ -3,10 +3,11 @@ module github.com/aquasecurity/trivy
 go 1.13
 
 require (
-	github.com/aquasecurity/fanal v0.0.0-20191225115707-0939236aadb8
+	github.com/aquasecurity/fanal v0.0.0-20200105203922-a3284d4a2d11
 	github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b
 	github.com/aquasecurity/trivy-db v0.0.0-20191226181755-d6cabf5bc5d1
 	github.com/caarlos0/env/v6 v6.0.0
+	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/cheggaaa/pb/v3 v3.0.3
 	github.com/docker/docker v0.0.0-20180924202107-a9c061deec0f
 	github.com/genuinetools/reg v0.16.0

go.sum

@@ -29,8 +29,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/aquasecurity/fanal v0.0.0-20190819081512-f04452b627c6/go.mod h1:enEz4FFetw4XAbkffaYgyCVq1556R9Ry+noqT4rq9BE=
-github.com/aquasecurity/fanal v0.0.0-20191225115707-0939236aadb8 h1:d55FvX5b+36P1Q1dV7UoV7rttn3eghmEPlW+NPmgQ0k=
-github.com/aquasecurity/fanal v0.0.0-20191225115707-0939236aadb8/go.mod h1:I+y3thuZSKnPXWqIOFDH4sk+ekiJ8V/XB71PDzhQxL8=
+github.com/aquasecurity/fanal v0.0.0-20200105203922-a3284d4a2d11 h1:TDDvXf5j45gdgbFLMQO4XDHMF2bHYKJ3CJnOy4L43So=
+github.com/aquasecurity/fanal v0.0.0-20200105203922-a3284d4a2d11/go.mod h1:I+y3thuZSKnPXWqIOFDH4sk+ekiJ8V/XB71PDzhQxL8=
 github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b h1:55Ulc/gvfWm4ylhVaR7MxOwujRjA6et7KhmUbSgUFf4=
 github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b/go.mod h1:BpNTD9vHfrejKsED9rx04ldM1WIbeyXGYxUrqTVwxVQ=
 github.com/aquasecurity/trivy v0.1.6/go.mod h1:5hobyhxLzDtxruHzPxpND2PUKOssvGUdE9BocpJUwo4=
@@ -52,6 +52,8 @@ github.com/briandowns/spinner v0.0.0-20190319032542-ac46072a5a91 h1:GMmnK0dvr0Sf
 github.com/briandowns/spinner v0.0.0-20190319032542-ac46072a5a91/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU=
 github.com/caarlos0/env/v6 v6.0.0 h1:NZt6FAoB8ieKO5lEwRdwCzYxWFx7ZYF2R7UcoyaWtyc=
 github.com/caarlos0/env/v6 v6.0.0/go.mod h1:+wdyOmtjoZIW2GJOc2OYa5NoOFuWD/bIpWqm30NgtRk=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cheggaaa/pb/v3 v3.0.3 h1:8WApbyUmgMOz7WIxJVNK0IRDcRfAmTxcEdi0TuxjdP4=
 github.com/cheggaaa/pb/v3 v3.0.3/go.mod h1:Pp35CDuiEpHa/ZLGCtBbM6CBwMstv1bJlG884V+73Yc=

integration/docker_engine_test.go

@@ -38,70 +38,71 @@ func TestRun_WithDockerEngine(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		ctx := context.Background()
-		defer ctx.Done()
+		t.Run(tc.name, func(t *testing.T) {
+			// Copy DB file
+			cacheDir := gunzipDB()
+			defer os.RemoveAll(cacheDir)
 
-		cli, err := client.NewClientWithOpts(client.FromEnv)
-		require.NoError(t, err, tc.name)
+			ctx := context.Background()
+			defer ctx.Done()
 
-		if !tc.invalidImage {
-			testfile, err := os.Open(tc.testfile)
+			cli, err := client.NewClientWithOpts(client.FromEnv)
 			require.NoError(t, err, tc.name)
 
-			// ensure image doesnt already exists
-			_, _ = cli.ImageRemove(ctx, tc.testfile, types.ImageRemoveOptions{
-				Force:         true,
-				PruneChildren: true,
-			})
+			if !tc.invalidImage {
+				testfile, err := os.Open(tc.testfile)
+				require.NoError(t, err, tc.name)
 
-			// load image into docker engine
-			_, err = cli.ImageLoad(ctx, testfile, true)
+				// ensure image doesnt already exists
+				_, _ = cli.ImageRemove(ctx, tc.testfile, types.ImageRemoveOptions{
+					Force:         true,
+					PruneChildren: true,
+				})
+
+				// load image into docker engine
+				_, err = cli.ImageLoad(ctx, testfile, true)
+				require.NoError(t, err, tc.name)
+
+				// tag our image to something unique
+				err = cli.ImageTag(ctx, "alpine:3.10", tc.testfile)
+				require.NoError(t, err, tc.name)
+			}
+
+			of, err := ioutil.TempFile("", "integration-docker-engine-output-file-*")
 			require.NoError(t, err, tc.name)
+			defer os.Remove(of.Name())
 
-			// tag our image to something unique
-			err = cli.ImageTag(ctx, "alpine:3.10", tc.testfile)
-			require.NoError(t, err, tc.name)
-		}
+			// run trivy
+			app := internal.NewApp("dev")
+			trivyArgs := []string{"trivy", "--skip-update", "--cache-dir", cacheDir, "--format=json"}
+			if !tc.invalidImage {
+				trivyArgs = append(trivyArgs, "--output", of.Name())
+			}
+			trivyArgs = append(trivyArgs, tc.testfile)
 
-		// run trivy
-		tmpDir, err := ioutil.TempDir("", "integration-docker-engine-*")
-		require.NoError(t, err)
-		defer func() {
-			os.RemoveAll(tmpDir)
-		}()
+			err = app.Run(trivyArgs)
+			switch {
+			case tc.expectedError != "":
+				assert.Equal(t, tc.expectedError, err.Error(), tc.name)
+			default:
+				assert.NoError(t, err, tc.name)
+			}
 
-		of, err := ioutil.TempFile(tmpDir, "integration-docker-engine-output-file-*")
-		require.NoError(t, err, tc.name)
-		app := internal.NewApp("dev")
+			if !tc.invalidImage {
+				// check for vulnerability output info
+				got, err := ioutil.ReadAll(of)
+				assert.NoError(t, err, tc.name)
+				want, err := ioutil.ReadFile(tc.expectedOutputFile)
+				assert.NoError(t, err, tc.name)
+				assert.JSONEq(t, string(want), string(got), tc.name)
 
-		trivyArgs := []string{"--skip-update", "--quiet", "--cache-dir", tmpDir, "--format=json"}
-		if !tc.invalidImage {
-			trivyArgs = append(trivyArgs, "--output", of.Name())
-		}
-		trivyArgs = append(trivyArgs, tc.testfile)
-
-		err = app.Run(trivyArgs)
-		switch {
-		case tc.expectedError != "":
-			assert.Equal(t, tc.expectedError, err.Error(), tc.name)
-		default:
-			assert.NoError(t, err, tc.name)
-		}
-
-		if !tc.invalidImage {
-			// check for vulnerability output info
-			got, err := ioutil.ReadAll(of)
-			assert.NoError(t, err, tc.name)
-			want, err := ioutil.ReadFile(tc.expectedOutputFile)
-			assert.NoError(t, err, tc.name)
-			assert.JSONEq(t, string(want), string(got), tc.name)
-
-			// cleanup
-			_, err = cli.ImageRemove(ctx, tc.testfile, types.ImageRemoveOptions{
-				Force:         true,
-				PruneChildren: true,
-			})
-			assert.NoError(t, err, tc.name)
-		}
+				// cleanup
+				_, err = cli.ImageRemove(ctx, tc.testfile, types.ImageRemoveOptions{
+					Force:         true,
+					PruneChildren: true,
+				})
+				assert.NoError(t, err, tc.name)
+			}
+		})
 	}
 }

pkg/rpc/client/library/client.go

@@ -4,14 +4,13 @@ import (
 	"context"
 	"net/http"
 
-	"github.com/aquasecurity/trivy/pkg/rpc/client"
-
 	"github.com/google/wire"
 	"golang.org/x/xerrors"
 
 	ptypes "github.com/aquasecurity/go-dep-parser/pkg/types"
 	detector "github.com/aquasecurity/trivy/pkg/detector/library"
 	r "github.com/aquasecurity/trivy/pkg/rpc"
+	"github.com/aquasecurity/trivy/pkg/rpc/client"
 	"github.com/aquasecurity/trivy/pkg/types"
 	rpc "github.com/aquasecurity/trivy/rpc/detector"
 )
@@ -41,9 +40,15 @@ func NewDetector(customHeaders CustomHeaders, detector rpc.LibDetector) Detector
 
 func (d Detector) Detect(filePath string, libs []ptypes.Library) ([]types.DetectedVulnerability, error) {
 	ctx := client.WithCustomHeaders(context.Background(), http.Header(d.customHeaders))
-	res, err := d.client.Detect(ctx, &rpc.LibDetectRequest{
-		FilePath:  filePath,
-		Libraries: r.ConvertToRpcLibraries(libs),
+
+	var res *rpc.DetectResponse
+	err := r.Retry(func() error {
+		var err error
+		res, err = d.client.Detect(ctx, &rpc.LibDetectRequest{
+			FilePath:  filePath,
+			Libraries: r.ConvertToRpcLibraries(libs),
+		})
+		return err
 	})
 	if err != nil {
 		return nil, xerrors.Errorf("failed to detect vulnerabilities via RPC: %w", err)

pkg/rpc/client/ospkg/client.go

@@ -4,14 +4,13 @@ import (
 	"context"
 	"net/http"
 
-	"github.com/aquasecurity/trivy/pkg/rpc/client"
-
 	"github.com/google/wire"
 	"golang.org/x/xerrors"
 
 	"github.com/aquasecurity/fanal/analyzer"
 	detector "github.com/aquasecurity/trivy/pkg/detector/ospkg"
 	r "github.com/aquasecurity/trivy/pkg/rpc"
+	"github.com/aquasecurity/trivy/pkg/rpc/client"
 	"github.com/aquasecurity/trivy/pkg/types"
 	rpc "github.com/aquasecurity/trivy/rpc/detector"
 )
@@ -41,10 +40,16 @@ func NewDetector(customHeaders CustomHeaders, detector rpc.OSDetector) Detector
 
 func (d Detector) Detect(osFamily, osName string, pkgs []analyzer.Package) ([]types.DetectedVulnerability, bool, error) {
 	ctx := client.WithCustomHeaders(context.Background(), http.Header(d.customHeaders))
-	res, err := d.client.Detect(ctx, &rpc.OSDetectRequest{
-		OsFamily: osFamily,
-		OsName:   osName,
-		Packages: r.ConvertToRpcPkgs(pkgs),
+
+	var res *rpc.DetectResponse
+	err := r.Retry(func() error {
+		var err error
+		res, err = d.client.Detect(ctx, &rpc.OSDetectRequest{
+			OsFamily: osFamily,
+			OsName:   osName,
+			Packages: r.ConvertToRpcPkgs(pkgs),
+		})
+		return err
 	})
 	if err != nil {
 		return nil, false, xerrors.Errorf("failed to detect vulnerabilities via RPC: %w", err)

pkg/rpc/retry.go

@@ -0,0 +1,40 @@
+package rpc
+
+import (
+	"time"
+
+	"github.com/aquasecurity/trivy/pkg/log"
+	"github.com/cenkalti/backoff"
+	"github.com/twitchtv/twirp"
+)
+
+const (
+	maxRetries = 10
+)
+
+func Retry(f func() error) error {
+	operation := func() error {
+		err := f()
+		if err != nil {
+			twerr, ok := err.(twirp.Error)
+			if !ok {
+				return backoff.Permanent(err)
+			}
+			if twerr.Code() == twirp.Unavailable {
+				return err
+			}
+			return backoff.Permanent(err)
+		}
+		return nil
+	}
+
+	b := backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries)
+	err := backoff.RetryNotify(operation, b, func(err error, _ time.Duration) {
+		log.Logger.Warn(err)
+		log.Logger.Info("Retrying HTTP request...")
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}