[INS-285] Fix custom detectors line number reporting to match the full regex instead of capture group

pkg/custom_detectors/CUSTOM_DETECTORS.md

@@ -38,7 +38,7 @@ This guide will walk you through setting up a custom detector in TruffleHog to i
    - **`verify`**: An optional section to validate detected secrets. If you want to verify or unverify detected secrets, this section needs to be configured. If not configured, all detected secrets will be marked as unverified. Read [verification server examples](#verification-server-examples)
 
    **Other allowed parameters:**
-   - **`primary_regex_name`**: This parameter allows you designate the primary regex pattern when multiple regex patterns are defined in the regex section. If a match is found, the match for the designated primary regex will be used to determine the line number. The value must be one of the names specified in the regex section.
+   - **`primary_regex_name`**: This parameter allows you designate the primary regex pattern when multiple regex patterns are defined in the regex section. If a match is found, the match for the designated primary regex will be used to determine the line number. The value must be one of the names specified in the regex section. If not provided, the first regex defined in the regex section will be used as the primary regex by default.
    - **`exclude_regexes_capture`**: This parameter allows you to define regex patterns to exclude specific parts of a detected secret. If a match is found within the detected secret, the portion matching this regex is excluded from the result.
    - **`exclude_regexes_match`**: This parameter enables you to define regex patterns to exclude entire matches from being reported as secrets. This applies to the entire matched string, not just the token.
    - **`entropy`**: This parameter is used to assess the randomness of detected strings. High entropy often indicates that a string is a potential secret, such as an API key or password, due to its complexity and unpredictability. It helps in filtering false-positives. While an entropy threshold of `3` can be a starting point, it's essential to adjust this value based on your project's specific requirements and the nature of the data you have.

pkg/custom_detectors/custom_detectors.go

@@ -66,6 +66,9 @@ func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*CustomRegexWebh
 		}
 	}
 
+	// Ensure primary regex name is set.
+	ensurePrimaryRegexNameSet(pb)
+
 	// TODO: Copy only necessary data out of pb.
 	return &CustomRegexWebhook{pb}, nil
 }
@@ -229,14 +232,16 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string
 		values := match[key]
 		// values[0] contains the entire regex match.
 		secret := values[0]
+		fullMatch := values[0]
 		if len(values) > 1 {
 			secret = values[1]
 		}
 		raw += secret
 
-		// if the match is of the primary regex, set it's value as primary secret value in result
+		// if the match is of the primary regex, set it's full match as primary secret value in result
 		if c.PrimaryRegexName == key {
-			result.SetPrimarySecretValue(secret)
+			// we're using the full match here to ensure correct line number reporting
+			result.SetPrimarySecretValue(fullMatch)
 		}
 	}
 
@@ -394,3 +399,14 @@ func (c *CustomRegexWebhook) Description() string {
 	}
 	return c.GetDescription()
 }
+
+// ensurePrimaryRegexNameSet sets the PrimaryRegexName field to the first
+// regex name if it is not already set.
+func ensurePrimaryRegexNameSet(pb *custom_detectorspb.CustomRegex) {
+	if pb.PrimaryRegexName == "" {
+		for name := range pb.Regex {
+			pb.PrimaryRegexName = name
+			return
+		}
+	}
+}

pkg/custom_detectors/custom_detectors_test.go

@@ -232,6 +232,61 @@ func TestDetectorPrimarySecret(t *testing.T) {
 	assert.Equal(t, "secret_YI7C90ACY1_yy", results[0].GetPrimarySecretValue())
 }
 
+func TestDetectorPrimarySecretFullMatch(t *testing.T) {
+	tests := []struct {
+		name  string
+		input *custom_detectorspb.CustomRegex
+		chunk []byte
+		want  string
+	}{
+		{
+			name: "primary regex full match",
+			input: &custom_detectorspb.CustomRegex{
+				Name:             "test",
+				Keywords:         []string{"secret"},
+				Regex:            map[string]string{"secret": `secret *= *"([^"\r\n]+)"`},
+				PrimaryRegexName: "secret",
+			},
+			chunk: []byte(`
+			// some code
+			secret="mysecret"
+			// some code
+			`),
+			want: `secret="mysecret"`,
+		},
+		// Write a test case for a match which span multiple lines.
+		{
+			name: "primary regex full match multiline",
+			input: &custom_detectorspb.CustomRegex{
+				Name:             "test",
+				Keywords:         []string{"secret"},
+				Regex:            map[string]string{"secret": `secret *= *"([^"]+)"`},
+				PrimaryRegexName: "secret",
+			},
+			chunk: []byte(`
+			// some code
+			secret="mysecret
+			thatspansmultiplelines"
+			// some code
+			`),
+			want: `secret="mysecret
+			thatspansmultiplelines"`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			detector, err := NewWebhookCustomRegex(tt.input)
+			assert.NoError(t, err)
+			results, err := detector.FromData(context.Background(), false, tt.chunk)
+			assert.NoError(t, err)
+			assert.Equal(t, 1, len(results))
+			assert.Equal(t, tt.want, results[0].GetPrimarySecretValue())
+		})
+	}
+
+}
+
 func TestDetectorValidations(t *testing.T) {
 	type args struct {
 		CustomRegex *custom_detectorspb.CustomRegex
@@ -707,6 +762,24 @@ func TestNewWebhookCustomRegex_Validation(t *testing.T) {
 	}
 }
 
+func TestNewWebhookCustomRegex_EnsurePrimaryRegexNameSet(t *testing.T) {
+	t.Parallel()
+
+	pb := &custom_detectorspb.CustomRegex{
+		Name:     "test",
+		Keywords: []string{"kw"},
+		Regex: map[string]string{
+			"first":  `first_regex`,
+			"second": `second_regex`,
+		},
+		// PrimaryRegexName is not set.
+	}
+
+	detector, err := NewWebhookCustomRegex(pb)
+	assert.NoError(t, err)
+	assert.Equal(t, "first", detector.GetPrimaryRegexName(), "expected PrimaryRegexName to be set to the first regex name")
+}
+
 func BenchmarkProductIndices(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		_ = productIndices(3, 2, 6)