diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index cc72b1d5863d..7c40b0bccc1c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -142,6 +142,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Functionbeat* +- New options to configure roles and VPC. {pull}11779[11779] + *Winlogbeat* - Add support for reading from .evtx files. {issue}4450[4450] diff --git a/x-pack/functionbeat/_meta/beat.reference.yml b/x-pack/functionbeat/_meta/beat.reference.yml index 4b2e9ef22802..362c97712ff9 100644 --- a/x-pack/functionbeat/_meta/beat.reference.yml +++ b/x-pack/functionbeat/_meta/beat.reference.yml @@ -34,6 +34,14 @@ functionbeat.provider.aws.functions: # There is a hard limit of 3008MiB for each function. Default is 128MiB. #memory_size: 128MiB + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: @@ -71,6 +79,14 @@ functionbeat.provider.aws.functions: # There is a hard limit of 3008MiB for each function. Default is 128MiB. #memory_size: 128MiB + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: @@ -113,6 +129,14 @@ functionbeat.provider.aws.functions: # There is a hard limit of 3008MiB for each function. Default is 128MiB. #memory_size: 128MiB + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: diff --git a/x-pack/functionbeat/_meta/beat.yml b/x-pack/functionbeat/_meta/beat.yml index 2274a5033c82..0c8f4a37d14c 100644 --- a/x-pack/functionbeat/_meta/beat.yml +++ b/x-pack/functionbeat/_meta/beat.yml @@ -38,6 +38,14 @@ functionbeat.provider.aws.functions: # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Optional fields that you can specify to add additional information to the # output. Fields can be scalar values, arrays, dictionaries, or any nested # combination of these. @@ -75,6 +83,14 @@ functionbeat.provider.aws.functions: # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Optional fields that you can specify to add additional information to the # output. Fields can be scalar values, arrays, dictionaries, or any nested # combination of these. @@ -117,6 +133,14 @@ functionbeat.provider.aws.functions: # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Optional fields that you can specify to add additional information to the # output. Fields can be scalar values, arrays, dictionaries, or any nested # combination of these. diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index df275c6a6e5d..d35fbaf6154a 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -34,6 +34,14 @@ functionbeat.provider.aws.functions: # There is a hard limit of 3008MiB for each function. Default is 128MiB. #memory_size: 128MiB + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: @@ -71,6 +79,14 @@ functionbeat.provider.aws.functions: # There is a hard limit of 3008MiB for each function. Default is 128MiB. #memory_size: 128MiB + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: @@ -113,6 +129,14 @@ functionbeat.provider.aws.functions: # There is a hard limit of 3008MiB for each function. Default is 128MiB. #memory_size: 128MiB + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: diff --git a/x-pack/functionbeat/functionbeat.yml b/x-pack/functionbeat/functionbeat.yml index c819d782c93d..bfbd9167ab3e 100644 --- a/x-pack/functionbeat/functionbeat.yml +++ b/x-pack/functionbeat/functionbeat.yml @@ -38,6 +38,14 @@ functionbeat.provider.aws.functions: # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Optional fields that you can specify to add additional information to the # output. Fields can be scalar values, arrays, dictionaries, or any nested # combination of these. @@ -75,6 +83,14 @@ functionbeat.provider.aws.functions: # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Optional fields that you can specify to add additional information to the # output. Fields can be scalar values, arrays, dictionaries, or any nested # combination of these. @@ -117,6 +133,14 @@ functionbeat.provider.aws.functions: # Dead letter queue configuration, this must be set to an ARN pointing to a SQS queue. #dead_letter_config.target_arn: + # Execution role of the function. + #role: arn:aws:iam::123456789012:role/MyFunction + + # Connect to private resources in an Amazon VPC. + #virtual_private_cloud: + # security_group_ids: [] + # subnet_ids: [] + # Optional fields that you can specify to add additional information to the # output. Fields can be scalar values, arrays, dictionaries, or any nested # combination of these. diff --git a/x-pack/functionbeat/provider/aws/cli_manager.go b/x-pack/functionbeat/provider/aws/cli_manager.go index acac77015478..eed1be860fa5 100644 --- a/x-pack/functionbeat/provider/aws/cli_manager.go +++ b/x-pack/functionbeat/provider/aws/cli_manager.go @@ -85,50 +85,17 @@ func (c *CLIManager) template(function installer, name, codeLoc string) *cloudfo // Documentation: https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/Welcome.html // Intrinsic function reference: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference.html - // Default policies to writes logs from the Lambda. - policies := []cloudformation.AWSIAMRole_Policy{ - cloudformation.AWSIAMRole_Policy{ - PolicyName: cloudformation.Join("-", []string{"fnb", "lambda", name}), - PolicyDocument: map[string]interface{}{ - "Statement": []map[string]interface{}{ - map[string]interface{}{ - "Action": []string{"logs:CreateLogStream", "Logs:PutLogEvents"}, - "Effect": "Allow", - "Resource": []string{ - cloudformation.Sub("arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/" + name + ":*"), - }, - }, - }, - }, - }, - } + template := cloudformation.NewTemplate() - // Merge any specific policies from the service. - policies = append(policies, function.Policies()...) + role := lambdaConfig.Role + dependsOn := make([]string, 0) + if lambdaConfig.Role == "" { + c.log.Infof("No role is configured for function %s, creating a custom role.", name) - // Create the roles for the lambda. - template := cloudformation.NewTemplate() - // doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html - template.Resources[prefix("")+"IAMRoleLambdaExecution"] = &cloudformation.AWSIAMRole{ - AssumeRolePolicyDocument: map[string]interface{}{ - "Statement": []interface{}{ - map[string]interface{}{ - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": map[string]interface{}{ - "Service": cloudformation.Join("", []string{ - "lambda.", - cloudformation.Ref("AWS::URLSuffix"), - }), - }, - }, - }, - }, - Path: "/", - RoleName: "functionbeat-lambda-" + name, - // Allow the lambda to write log to cloudwatch logs. - // doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html - Policies: policies, + roleRes := prefix("") + "IAMRoleLambdaExecution" + template.Resources[roleRes] = c.roleTemplate(function, name) + role = cloudformation.GetAtt(roleRes, "Arn") + dependsOn = []string{roleRes} } // Configure the Dead letter, any failed events will be send to the configured amazon resource name. @@ -139,6 +106,15 @@ func (c *CLIManager) template(function installer, name, codeLoc string) *cloudfo } } + // Configure VPC + var vcpConf *cloudformation.AWSLambdaFunction_VpcConfig + if lambdaConfig.VPCConfig != nil && len(lambdaConfig.VPCConfig.SecurityGroupIDs) != 0 && len(lambdaConfig.VPCConfig.SubnetIDs) != 0 { + vcpConf = &cloudformation.AWSLambdaFunction_VpcConfig{ + SecurityGroupIds: lambdaConfig.VPCConfig.SecurityGroupIDs, + SubnetIds: lambdaConfig.VPCConfig.SubnetIDs, + } + } + // Create the lambda // Doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-function.html template.Resources[prefix("")] = &AWSLambdaFunction{ @@ -156,15 +132,16 @@ func (c *CLIManager) template(function installer, name, codeLoc string) *cloudfo }, }, DeadLetterConfig: dlc, + VpcConfig: vcpConf, FunctionName: name, - Role: cloudformation.GetAtt(prefix("")+"IAMRoleLambdaExecution", "Arn"), + Role: role, Runtime: runtime, Handler: handlerName, MemorySize: lambdaConfig.MemorySize.Megabytes(), ReservedConcurrentExecutions: lambdaConfig.Concurrency, Timeout: int(lambdaConfig.Timeout.Seconds()), }, - DependsOn: []string{prefix("") + "IAMRoleLambdaExecution"}, + DependsOn: dependsOn, } // Create the log group for the specific function lambda. @@ -175,6 +152,53 @@ func (c *CLIManager) template(function installer, name, codeLoc string) *cloudfo return template } +func (c *CLIManager) roleTemplate(function installer, name string) *cloudformation.AWSIAMRole { + // Default policies to writes logs from the Lambda. + policies := []cloudformation.AWSIAMRole_Policy{ + cloudformation.AWSIAMRole_Policy{ + PolicyName: cloudformation.Join("-", []string{"fnb", "lambda", name}), + PolicyDocument: map[string]interface{}{ + "Statement": []map[string]interface{}{ + map[string]interface{}{ + "Action": []string{"logs:CreateLogStream", "logs:PutLogEvents"}, + "Effect": "Allow", + "Resource": []string{ + cloudformation.Sub("arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/" + name + ":*"), + }, + }, + }, + }, + }, + } + + // Merge any specific policies from the service. + policies = append(policies, function.Policies()...) + + // Create the roles for the lambda. + // doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html + return &cloudformation.AWSIAMRole{ + AssumeRolePolicyDocument: map[string]interface{}{ + "Statement": []interface{}{ + map[string]interface{}{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": map[string]interface{}{ + "Service": cloudformation.Join("", []string{ + "lambda.", + cloudformation.Ref("AWS::URLSuffix"), + }), + }, + }, + }, + }, + Path: "/", + RoleName: "functionbeat-lambda-" + name, + // Allow the lambda to write log to cloudwatch logs. + // doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html + Policies: policies, + } +} + // stackName cloudformation stack are unique per function. func (c *CLIManager) stackName(name string) string { return "fnb-" + name + "-stack" diff --git a/x-pack/functionbeat/provider/aws/cloudwatch_logs_test.go b/x-pack/functionbeat/provider/aws/cloudwatch_logs_test.go index eb0dc1e0c219..0d6d41a3da0e 100644 --- a/x-pack/functionbeat/provider/aws/cloudwatch_logs_test.go +++ b/x-pack/functionbeat/provider/aws/cloudwatch_logs_test.go @@ -49,7 +49,7 @@ func TestCloudwatchLogs(t *testing.T) { cfg := common.MustNewConfigFrom(map[string]interface{}{ "name": "foobar", "description": "my long description", - "role": "arn:aws:iam::00000000:role/functionbeat", + "role": "arn:aws:iam::000000000000:role/functionbeat", "triggers": []map[string]interface{}{ map[string]interface{}{ "log_group_name": "foo", diff --git a/x-pack/functionbeat/provider/aws/config.go b/x-pack/functionbeat/provider/aws/config.go index 2024fe3e7b2f..5eb5f8fc4e5a 100644 --- a/x-pack/functionbeat/provider/aws/config.go +++ b/x-pack/functionbeat/provider/aws/config.go @@ -6,6 +6,7 @@ package aws import ( "fmt" + "regexp" "time" "unicode" @@ -23,11 +24,17 @@ type Config struct { const maxMegabytes = 3008 // DefaultLambdaConfig confguration for AWS lambda function. -var DefaultLambdaConfig = &lambdaConfig{ - MemorySize: 128 * 1024 * 1024, - Timeout: time.Second * 3, - Concurrency: 5, -} +var ( + DefaultLambdaConfig = &lambdaConfig{ + MemorySize: 128 * 1024 * 1024, + Timeout: time.Second * 3, + Concurrency: 5, + } + + // Source: https://docs.aws.amazon.com/lambda/latest/dg/API_CreateFunction.html#SSS-CreateFunction-request-Role + arnRolePattern = "arn:(aws[a-zA-Z-]*)?:iam::\\d{12}:role/?[a-zA-Z_0-9+=,.@\\-_/]+" + roleRE = regexp.MustCompile(arnRolePattern) +) type lambdaConfig struct { Concurrency int `config:"concurrency" validate:"min=0,max=1000"` @@ -35,6 +42,8 @@ type lambdaConfig struct { Description string `config:"description"` MemorySize MemSizeFactor64 `config:"memory_size"` Timeout time.Duration `config:"timeout" validate:"nonzero,positive"` + Role string `config:"role"` + VPCConfig *vpcConfig `config:"virtual_private_cloud"` } func (c *lambdaConfig) Validate() error { @@ -46,6 +55,10 @@ func (c *lambdaConfig) Validate() error { return fmt.Errorf("'memory_size' must be lower than %d", maxMegabytes) } + if c.Role != "" && !roleRE.MatchString(c.Role) { + return fmt.Errorf("invalid role: '%s', name must match pattern %s", c.Role, arnRolePattern) + } + return nil } @@ -53,6 +66,11 @@ type deadLetterConfig struct { TargetArn string `config:"target_arn"` } +type vpcConfig struct { + SecurityGroupIDs []string `config:"security_group_ids" validate:"required"` + SubnetIDs []string `config:"subnet_ids" validate:"required"` +} + // MemSizeFactor64 implements a human understandable format for bytes but also make sure that all // values used are a factory of 64. type MemSizeFactor64 int