[INS-255] Updated datadog detector to set verificationError in case of a verification error

pkg/detectors/datadogtoken/datadogtoken.go

@@ -3,6 +3,8 @@ package datadogtoken
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"io"
 	"net/http"
 	"strings"
 
@@ -14,6 +16,7 @@ import (
 )
 
 type Scanner struct {
+	client *http.Client
 	detectors.EndpointSetter
 	detectors.DefaultMultiPartCredentialProvider
 }
@@ -29,8 +32,9 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
-	apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
+	appPat        = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`)
+	apiPat        = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`)
+	datadogURLPat = regexp.MustCompile(`\b(api(?:\.[a-z0-9-]+)?\.(?:datadoghq|ddog-gov)\.[a-z]{2,3})\b`)
 )
 
 type userServiceResponse struct {
@@ -92,10 +96,17 @@ func setOrganizationInfo(opt []*options, s1 *detectors.Result) {
 
 }
 
+func (s Scanner) getClient() *http.Client {
+	if s.client != nil {
+		return s.client
+	}
+	return client
+}
+
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"datadog"}
+	return []string{"datadog", "ddog-gov"}
 }
 
 // FromData will find and optionally verify DatadogToken secrets in a given set of bytes.
@@ -105,12 +116,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
 	apiMatches := apiPat.FindAllStringSubmatch(dataStr, -1)
 
+	var uniqueFoundUrls = make(map[string]struct{})
+	for _, matches := range datadogURLPat.FindAllStringSubmatch(dataStr, -1) {
+		uniqueFoundUrls["https://"+matches[1]] = struct{}{}
+	}
+	endpoints := make([]string, 0, len(uniqueFoundUrls))
+	for endpoint := range uniqueFoundUrls {
+		endpoints = append(endpoints, endpoint)
+	}
+
 	for _, apiMatch := range apiMatches {
 		resApiMatch := strings.TrimSpace(apiMatch[1])
-		appIncluded := false
 		for _, appMatch := range appMatches {
 			resAppMatch := strings.TrimSpace(appMatch[1])
-
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_DatadogToken,
 				Raw:          []byte(resAppMatch),
@@ -121,64 +139,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
-				for _, baseURL := range s.Endpoints() {
-					req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v2/users", nil)
-					if err != nil {
-						continue
-					}
-					req.Header.Add("Content-Type", "application/json")
-					req.Header.Add("DD-API-KEY", resApiMatch)
-					req.Header.Add("DD-APPLICATION-KEY", resAppMatch)
-					res, err := client.Do(req)
-					if err == nil {
-						defer res.Body.Close()
-						if res.StatusCode >= 200 && res.StatusCode < 300 {
-							s1.Verified = true
-							s1.AnalysisInfo = map[string]string{"apiKey": resApiMatch, "appKey": resAppMatch}
-							var serviceResponse userServiceResponse
-							if err := json.NewDecoder(res.Body).Decode(&serviceResponse); err == nil {
-								// setup emails
-								if len(serviceResponse.Data) > 0 {
-									setUserEmails(serviceResponse.Data, &s1)
-								}
-								// setup organizations
-								if len(serviceResponse.Included) > 0 {
-									setOrganizationInfo(serviceResponse.Included, &s1)
-								}
-							}
+				for _, baseURL := range s.Endpoints(endpoints...) {
+					client := s.getClient()
+					res, isVerified, verificationErr := verifyMatch(ctx, client, resApiMatch, resAppMatch, baseURL)
+					s1.Verified = isVerified
+					s1.SetVerificationError(verificationErr, resApiMatch, resAppMatch)
+					if isVerified && res != nil {
+						s1.ResetVerificationError() // Reset verification error in case a secret is verified with an endpoint
+						s1.AnalysisInfo = map[string]string{"apiKey": resApiMatch, "appKey": resAppMatch, "endpoint": baseURL}
+						var serviceResponse userServiceResponse
+						if len(serviceResponse.Data) > 0 {
+							setUserEmails(serviceResponse.Data, &s1)
 						}
-					}
-				}
-			}
-			appIncluded = true
-			results = append(results, s1)
-		}
-
-		if !appIncluded {
-			s1 := detectors.Result{
-				DetectorType: detectorspb.DetectorType_DatadogToken,
-				Raw:          []byte(resApiMatch),
-				RawV2:        []byte(resApiMatch),
-				ExtraData: map[string]string{
-					"Type": "APIKeyOnly",
-				},
-			}
-
-			if verify {
-				for _, baseURL := range s.Endpoints() {
-					req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v1/validate", nil)
-					if err != nil {
-						continue
-					}
-					req.Header.Add("Content-Type", "application/json")
-					req.Header.Add("DD-API-KEY", resApiMatch)
-					res, err := client.Do(req)
-					if err == nil {
-						defer res.Body.Close()
-						if res.StatusCode >= 200 && res.StatusCode < 300 {
-							s1.Verified = true
-							s1.AnalysisInfo = map[string]string{"apiKey": resApiMatch}
+						if len(serviceResponse.Included) > 0 {
+							setOrganizationInfo(serviceResponse.Included, &s1)
 						}
+						// break the loop once we've successfully validated the token against a baseURL
+						break
 					}
 				}
 			}
@@ -196,3 +173,44 @@ func (s Scanner) Type() detectorspb.DetectorType {
 func (s Scanner) Description() string {
 	return "Datadog is a monitoring and security platform for cloud applications. Datadog API and Application keys can be used to access and manage data and configurations within Datadog."
 }
+
+func verifyMatch(ctx context.Context, client *http.Client, apiKey, appKey, baseUrl string) (*userServiceResponse, bool, error) {
+	// Reference: https://docs.datadoghq.com/api/latest/users/
+
+	req, err := http.NewRequestWithContext(ctx, "GET", baseUrl+"/api/v2/users", nil)
+	if err != nil {
+		return nil, false, err
+	}
+
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("DD-API-KEY", apiKey)
+	req.Header.Add("DD-APPLICATION-KEY", appKey)
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, false, err
+	}
+
+	defer func() {
+		_, _ = io.Copy(io.Discard, res.Body)
+		_ = res.Body.Close()
+	}()
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		var serviceResponse userServiceResponse
+		if err := json.NewDecoder(res.Body).Decode(&serviceResponse); err != nil {
+			return nil, false, err
+		}
+		return &serviceResponse, true, nil
+	case http.StatusBadRequest:
+		return nil, false, fmt.Errorf("bad request")
+	case http.StatusUnauthorized:
+		return nil, false, fmt.Errorf("invalid credentials")
+	case http.StatusForbidden:
+		return nil, false, fmt.Errorf("Insufficient permissions")
+	case http.StatusTooManyRequests:
+		return nil, false, fmt.Errorf("too many requests")
+	default:
+		return nil, false, fmt.Errorf("unexpected status code: %d", res.StatusCode)
+	}
+}

pkg/detectors/datadogtoken/datadogtoken_integration_test.go

@@ -25,7 +25,7 @@ func TestDatadogToken_FromChunk(t *testing.T) {
 	}
 	apiKey := testSecrets.MustGetField("DATADOGTOKEN_TOKEN")
 	appKey := testSecrets.MustGetField("DATADOGTOKEN_APPKEY")
-	inactiveAppKey := testSecrets.MustGetField("DATADOGTOKEN_INACTIVE")
+	endpoint := "https://api.us5.datadoghq.com"
 
 	type args struct {
 		ctx    context.Context
@@ -44,7 +44,7 @@ func TestDatadogToken_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, apiKey)),
+				data:   []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s and endpoint %s", appKey, apiKey, endpoint)),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -54,43 +54,10 @@ func TestDatadogToken_FromChunk(t *testing.T) {
 					ExtraData: map[string]string{
 						"Type": "Application+APIKey",
 					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a datadogtoken secret %s within but datadog %s not valid", inactiveAppKey, apiKey)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_DatadogToken,
-					Verified:     false,
-					ExtraData: map[string]string{
-						"Type": "Application+APIKey",
-					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "api key found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a datadogtoken secret %s", apiKey)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_DatadogToken,
-					Verified:     true,
-					ExtraData: map[string]string{
-						"Type": "APIKeyOnly",
+					AnalysisInfo: map[string]string{
+						"apiKey":   apiKey,
+						"appKey":   appKey,
+						"endpoint": endpoint,
 					},
 				},
 			},
@@ -115,6 +82,7 @@ func TestDatadogToken_FromChunk(t *testing.T) {
 			// use default cloud endpoint
 			s.UseCloudEndpoint(true)
 			s.SetCloudEndpoint(s.CloudEndpoint())
+			s.UseFoundEndpoints(true)
 
 			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
@@ -136,6 +104,61 @@ func TestDatadogToken_FromChunk(t *testing.T) {
 	}
 }
 
+func TestDatadogToken_FromChunk_Unverified(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+
+	apiKey := testSecrets.MustGetField("DATADOGTOKEN_TOKEN")
+	inactiveAppKey := testSecrets.MustGetField("DATADOGTOKEN_INACTIVE")
+
+	data := []byte(fmt.Sprintf(
+		"You can find a datadogtoken secret %s within but datadog %s not valid",
+		inactiveAppKey,
+		apiKey,
+	))
+
+	s := Scanner{}
+	s.UseCloudEndpoint(true)
+	s.SetCloudEndpoint(s.CloudEndpoint())
+	s.UseFoundEndpoints(true)
+
+	results, err := s.FromData(ctx, true, data)
+	if err != nil {
+		t.Fatalf("FromData returned error: %v", err)
+	}
+
+	if len(results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(results))
+	}
+
+	r := results[0]
+
+	if r.DetectorType != detectorspb.DetectorType_DatadogToken {
+		t.Errorf("unexpected detector type: %v", r.DetectorType)
+	}
+
+	if r.Verified {
+		t.Errorf("expected token to be unverified")
+	}
+
+	if r.VerificationError() == nil {
+		t.Errorf("Expected verification error")
+	}
+
+	if got := r.ExtraData["Type"]; got != "Application+APIKey" {
+		t.Errorf("unexpected ExtraData Type: %q", got)
+	}
+
+	if len(r.Raw) == 0 {
+		t.Errorf("expected raw secret to be present")
+	}
+}
+
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}