New options to configure roles and VPC (#11779)

From now on it is possible to configure permissions in `functionbeat.yml` for the deployed lambda function. Two new options are added: `role` and `virtual_private_cloud`.

```yaml
# Execution role of the function.
role: arn:aws:iam::123456789012:role/MyFunction
```
```yaml
# Connect to private resources in an Amazon VPC.
virtual_private_cloud:
  security_group_ids:
    - mySecurityGroup
    - anotherSecurityGroup
  subnet_ids:
    - myUniqueID
```

Note: I don't really like the name `virtual_private_cloud` as it's too long. But naming the option `vpc` seems wrong. Do you have any other suggestions?

Closes #9425

CHANGELOG.next.asciidoc

@@ -155,6 +155,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Functionbeat*
 
+- New options to configure roles and VPC. {pull}11779[11779]
+
 *Winlogbeat*
 
 - Add support for reading from .evtx files. {issue}4450[4450]

x-pack/functionbeat/_meta/beat.reference.yml

@@ -34,6 +34,14 @@ functionbeat.provider.aws.functions:
     # There is a hard limit of 3008MiB for each function. Default is 128MiB.
     #memory_size: 128MiB
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 
@@ -71,6 +79,14 @@ functionbeat.provider.aws.functions:
     # There is a hard limit of 3008MiB for each function. Default is 128MiB.
     #memory_size: 128MiB
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 
@@ -113,6 +129,14 @@ functionbeat.provider.aws.functions:
     # There is a hard limit of 3008MiB for each function. Default is 128MiB.
     #memory_size: 128MiB
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 

x-pack/functionbeat/_meta/beat.yml

@@ -38,6 +38,14 @@ functionbeat.provider.aws.functions:
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Optional fields that you can specify to add additional information to the
     # output. Fields can be scalar values, arrays, dictionaries, or any nested
     # combination of these.
@@ -75,6 +83,14 @@ functionbeat.provider.aws.functions:
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Optional fields that you can specify to add additional information to the
     # output. Fields can be scalar values, arrays, dictionaries, or any nested
     # combination of these.
@@ -117,6 +133,14 @@ functionbeat.provider.aws.functions:
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Optional fields that you can specify to add additional information to the
     # output. Fields can be scalar values, arrays, dictionaries, or any nested
     # combination of these.

x-pack/functionbeat/functionbeat.reference.yml

@@ -34,6 +34,14 @@ functionbeat.provider.aws.functions:
     # There is a hard limit of 3008MiB for each function. Default is 128MiB.
     #memory_size: 128MiB
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 
@@ -71,6 +79,14 @@ functionbeat.provider.aws.functions:
     # There is a hard limit of 3008MiB for each function. Default is 128MiB.
     #memory_size: 128MiB
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 
@@ -113,6 +129,14 @@ functionbeat.provider.aws.functions:
     # There is a hard limit of 3008MiB for each function. Default is 128MiB.
     #memory_size: 128MiB
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 

x-pack/functionbeat/functionbeat.yml

@@ -38,6 +38,14 @@ functionbeat.provider.aws.functions:
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Optional fields that you can specify to add additional information to the
     # output. Fields can be scalar values, arrays, dictionaries, or any nested
     # combination of these.
@@ -75,6 +83,14 @@ functionbeat.provider.aws.functions:
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Optional fields that you can specify to add additional information to the
     # output. Fields can be scalar values, arrays, dictionaries, or any nested
     # combination of these.
@@ -117,6 +133,14 @@ functionbeat.provider.aws.functions:
     # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue.
     #dead_letter_config.target_arn:
 
+    # Execution role of the function.
+    #role: arn:aws:iam::123456789012:role/MyFunction
+
+    # Connect to private resources in an Amazon VPC.
+    #virtual_private_cloud:
+    #  security_group_ids: []
+    #  subnet_ids: []
+
     # Optional fields that you can specify to add additional information to the
     # output. Fields can be scalar values, arrays, dictionaries, or any nested
     # combination of these.

x-pack/functionbeat/provider/aws/cli_manager.go

@@ -85,50 +85,17 @@ func (c *CLIManager) template(function installer, name, codeLoc string) *cloudfo
 	// Documentation: https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/Welcome.html
 	// Intrinsic function reference: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference.html
 
-	// Default policies to writes logs from the Lambda.
-	policies := []cloudformation.AWSIAMRole_Policy{
-		cloudformation.AWSIAMRole_Policy{
-			PolicyName: cloudformation.Join("-", []string{"fnb", "lambda", name}),
-			PolicyDocument: map[string]interface{}{
-				"Statement": []map[string]interface{}{
-					map[string]interface{}{
-						"Action": []string{"logs:CreateLogStream", "Logs:PutLogEvents"},
-						"Effect": "Allow",
-						"Resource": []string{
-							cloudformation.Sub("arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/" + name + ":*"),
-						},
-					},
-				},
-			},
-		},
-	}
+	template := cloudformation.NewTemplate()
 
-	// Merge any specific policies from the service.
-	policies = append(policies, function.Policies()...)
+	role := lambdaConfig.Role
+	dependsOn := make([]string, 0)
+	if lambdaConfig.Role == "" {
+		c.log.Infof("No role is configured for function %s, creating a custom role.", name)
 
-	// Create the roles for the lambda.
-	template := cloudformation.NewTemplate()
-	// doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html
-	template.Resources[prefix("")+"IAMRoleLambdaExecution"] = &cloudformation.AWSIAMRole{
-		AssumeRolePolicyDocument: map[string]interface{}{
-			"Statement": []interface{}{
-				map[string]interface{}{
-					"Action": "sts:AssumeRole",
-					"Effect": "Allow",
-					"Principal": map[string]interface{}{
-						"Service": cloudformation.Join("", []string{
-							"lambda.",
-							cloudformation.Ref("AWS::URLSuffix"),
-						}),
-					},
-				},
-			},
-		},
-		Path:     "/",
-		RoleName: "functionbeat-lambda-" + name,
-		// Allow the lambda to write log to cloudwatch logs.
-		// doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html
-		Policies: policies,
+		roleRes := prefix("") + "IAMRoleLambdaExecution"
+		template.Resources[roleRes] = c.roleTemplate(function, name)
+		role = cloudformation.GetAtt(roleRes, "Arn")
+		dependsOn = []string{roleRes}
 	}
 
 	// Configure the Dead letter, any failed events will be send to the configured amazon resource name.
@@ -139,6 +106,15 @@ func (c *CLIManager) template(function installer, name, codeLoc string) *cloudfo
 		}
 	}
 
+	// Configure VPC
+	var vcpConf *cloudformation.AWSLambdaFunction_VpcConfig
+	if lambdaConfig.VPCConfig != nil && len(lambdaConfig.VPCConfig.SecurityGroupIDs) != 0 && len(lambdaConfig.VPCConfig.SubnetIDs) != 0 {
+		vcpConf = &cloudformation.AWSLambdaFunction_VpcConfig{
+			SecurityGroupIds: lambdaConfig.VPCConfig.SecurityGroupIDs,
+			SubnetIds:        lambdaConfig.VPCConfig.SubnetIDs,
+		}
+	}
+
 	// Create the lambda
 	// Doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-function.html
 	template.Resources[prefix("")] = &AWSLambdaFunction{
@@ -156,15 +132,16 @@ func (c *CLIManager) template(function installer, name, codeLoc string) *cloudfo
 				},
 			},
 			DeadLetterConfig:             dlc,
+			VpcConfig:                    vcpConf,
 			FunctionName:                 name,
-			Role:                         cloudformation.GetAtt(prefix("")+"IAMRoleLambdaExecution", "Arn"),
+			Role:                         role,
 			Runtime:                      runtime,
 			Handler:                      handlerName,
 			MemorySize:                   lambdaConfig.MemorySize.Megabytes(),
 			ReservedConcurrentExecutions: lambdaConfig.Concurrency,
 			Timeout:                      int(lambdaConfig.Timeout.Seconds()),
 		},
-		DependsOn: []string{prefix("") + "IAMRoleLambdaExecution"},
+		DependsOn: dependsOn,
 	}
 
 	// Create the log group for the specific function lambda.
@@ -175,6 +152,53 @@ func (c *CLIManager) template(function installer, name, codeLoc string) *cloudfo
 	return template
 }
 
+func (c *CLIManager) roleTemplate(function installer, name string) *cloudformation.AWSIAMRole {
+	// Default policies to writes logs from the Lambda.
+	policies := []cloudformation.AWSIAMRole_Policy{
+		cloudformation.AWSIAMRole_Policy{
+			PolicyName: cloudformation.Join("-", []string{"fnb", "lambda", name}),
+			PolicyDocument: map[string]interface{}{
+				"Statement": []map[string]interface{}{
+					map[string]interface{}{
+						"Action": []string{"logs:CreateLogStream", "logs:PutLogEvents"},
+						"Effect": "Allow",
+						"Resource": []string{
+							cloudformation.Sub("arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/" + name + ":*"),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	// Merge any specific policies from the service.
+	policies = append(policies, function.Policies()...)
+
+	// Create the roles for the lambda.
+	// doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html
+	return &cloudformation.AWSIAMRole{
+		AssumeRolePolicyDocument: map[string]interface{}{
+			"Statement": []interface{}{
+				map[string]interface{}{
+					"Action": "sts:AssumeRole",
+					"Effect": "Allow",
+					"Principal": map[string]interface{}{
+						"Service": cloudformation.Join("", []string{
+							"lambda.",
+							cloudformation.Ref("AWS::URLSuffix"),
+						}),
+					},
+				},
+			},
+		},
+		Path:     "/",
+		RoleName: "functionbeat-lambda-" + name,
+		// Allow the lambda to write log to cloudwatch logs.
+		// doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html
+		Policies: policies,
+	}
+}
+
 // stackName cloudformation stack are unique per function.
 func (c *CLIManager) stackName(name string) string {
 	return "fnb-" + name + "-stack"

x-pack/functionbeat/provider/aws/cloudwatch_logs_test.go

@@ -49,7 +49,7 @@ func TestCloudwatchLogs(t *testing.T) {
 	cfg := common.MustNewConfigFrom(map[string]interface{}{
 		"name":        "foobar",
 		"description": "my long description",
-		"role":        "arn:aws:iam::00000000:role/functionbeat",
+		"role":        "arn:aws:iam::000000000000:role/functionbeat",
 		"triggers": []map[string]interface{}{
 			map[string]interface{}{
 				"log_group_name": "foo",