From c2e6d8a5e5df7aabfb04445948ce02745352a29e Mon Sep 17 00:00:00 2001 From: Mazen Selim Date: Thu, 27 Feb 2025 07:34:46 +0000 Subject: [PATCH] Build private & public subnet in every AZ --- internal/deployers/eksapi/infra.go | 79 ++--- .../{infra.yaml => infra.yaml.template} | 329 ++++-------------- .../deployers/eksapi/templates/templates.go | 14 +- internal/deployers/eksapi/vpc.go | 70 ++++ internal/deployers/eksapi/vpc_test.go | 56 +++ 5 files changed, 231 insertions(+), 317 deletions(-) rename internal/deployers/eksapi/templates/{infra.yaml => infra.yaml.template} (59%) create mode 100644 internal/deployers/eksapi/vpc.go create mode 100644 internal/deployers/eksapi/vpc_test.go diff --git a/internal/deployers/eksapi/infra.go b/internal/deployers/eksapi/infra.go index 3cc714a1f..a41326630 100644 --- a/internal/deployers/eksapi/infra.go +++ b/internal/deployers/eksapi/infra.go @@ -1,12 +1,12 @@ package eksapi import ( + "bytes" "context" _ "embed" "errors" "fmt" "path" - "slices" "strings" "time" @@ -89,47 +89,38 @@ func (i *Infrastructure) subnets() []string { func (m *InfrastructureManager) createInfrastructureStack(opts *deployerOptions) (*Infrastructure, error) { // TODO: create a subnet in every AZ // get two AZs for the subnets - azs, err := m.clients.EC2().DescribeAvailabilityZones(context.TODO(), &ec2.DescribeAvailabilityZonesInput{}) + describeAZsOuptut, err := m.clients.EC2().DescribeAvailabilityZones(context.TODO(), &ec2.DescribeAvailabilityZonesInput{}) if err != nil { return nil, err } - var subnetAzs []string - if opts.CapacityReservation { - subnetAzs, err = m.getAZsWithCapacity(opts) - if err != nil { - return nil, err - } - for _, az := range azs.AvailabilityZones { - if len(subnetAzs) == 2 { - break - } - if !slices.Contains(subnetAzs, *az.ZoneName) { - subnetAzs = append(subnetAzs, *az.ZoneName) - } - } - } else { - for i := 0; i < 2; i++ { - subnetAzs = append(subnetAzs, *azs.AvailabilityZones[i].ZoneName) - } + if describeAZsOuptut == nil || len(describeAZsOuptut.AvailabilityZones) == 0 { + return nil, fmt.Errorf("no availability zones returned from describe call") + } + ec2AvailabilityZones := describeAZsOuptut.AvailabilityZones + numAZs := len(describeAZsOuptut.AvailabilityZones) + var availabilityZones []string + for i := 0; i < numAZs; i++ { + availabilityZones = append(availabilityZones, aws.ToString(ec2AvailabilityZones[i].ZoneName)) + } + vpcConfig, err := getVpcConfig(availabilityZones) + if err != nil { + return nil, err + } + templateBuf := bytes.Buffer{} + err = templates.Infrastructure.Execute(&templateBuf, vpcConfig) + if err != nil { + return nil, err } - klog.Infof("creating infrastructure stack with AZs: %v", subnetAzs) + klog.Infof("creating infrastructure stack with AZs: %v", availabilityZones) input := cloudformation.CreateStackInput{ StackName: aws.String(m.resourceID), - TemplateBody: aws.String(templates.Infrastructure), + TemplateBody: aws.String(templateBuf.String()), Capabilities: []cloudformationtypes.Capability{cloudformationtypes.CapabilityCapabilityIam}, Parameters: []cloudformationtypes.Parameter{ { ParameterKey: aws.String("ResourceId"), ParameterValue: aws.String(m.resourceID), }, - { - ParameterKey: aws.String("Subnet01AZ"), - ParameterValue: aws.String(subnetAzs[0]), - }, - { - ParameterKey: aws.String("Subnet02AZ"), - ParameterValue: aws.String(subnetAzs[1]), - }, { ParameterKey: aws.String("AutoMode"), ParameterValue: aws.String(fmt.Sprintf("%t", opts.AutoMode)), @@ -167,7 +158,7 @@ func (m *InfrastructureManager) createInfrastructureStack(opts *deployerOptions) } klog.Infof("getting infrastructure stack resources: %s", *out.StackId) infra, err := m.getInfrastructureStackResources() - infra.availabilityZones = subnetAzs + infra.availabilityZones = availabilityZones if err != nil { return nil, fmt.Errorf("failed to get infrastructure stack resources: %w", err) } @@ -375,29 +366,3 @@ func (m *InfrastructureManager) getVPCCNINetworkInterfaceIds(vpcId string) ([]st } return enis, nil } - -func (m *InfrastructureManager) getAZsWithCapacity(opts *deployerOptions) ([]string, error) { - var subnetAzs []string - capacityReservations, err := m.clients.EC2().DescribeCapacityReservations(context.TODO(), &ec2.DescribeCapacityReservationsInput{ - Filters: []ec2types.Filter{ - { - Name: aws.String("instance-type"), - Values: opts.InstanceTypes, - }, - { - Name: aws.String("state"), - Values: []string{"active"}, - }, - }, - }) - if err != nil { - return nil, err - } - for _, cr := range capacityReservations.CapacityReservations { - if *cr.AvailableInstanceCount >= int32(opts.Nodes) { - subnetAzs = append(subnetAzs, *cr.AvailabilityZone) - break - } - } - return subnetAzs, nil -} diff --git a/internal/deployers/eksapi/templates/infra.yaml b/internal/deployers/eksapi/templates/infra.yaml.template similarity index 59% rename from internal/deployers/eksapi/templates/infra.yaml rename to internal/deployers/eksapi/templates/infra.yaml.template index a0162465e..9332d9e79 100644 --- a/internal/deployers/eksapi/templates/infra.yaml +++ b/internal/deployers/eksapi/templates/infra.yaml.template @@ -3,31 +3,6 @@ AWSTemplateFormatVersion: "2010-09-09" Description: "kubetest2-eksapi infrastructure" Parameters: - VpcBlock: - Type: String - Default: 192.168.0.0/16 - Description: The CIDR range for the VPC. This should be a valid private (RFC 1918) CIDR range. - - PublicSubnet01Block: - Type: String - Default: 192.168.0.0/18 - Description: CidrBlock for public subnet 01 within the VPC - - PublicSubnet02Block: - Type: String - Default: 192.168.64.0/18 - Description: CidrBlock for public subnet 02 within the VPC - - PrivateSubnet01Block: - Type: String - Default: 192.168.128.0/18 - Description: CidrBlock for private subnet 01 within the VPC - - PrivateSubnet02Block: - Type: String - Default: 192.168.192.0/18 - Description: CidrBlock for private subnet 02 within the VPC - AdditionalClusterRoleServicePrincipal: Type: String Default: "" @@ -36,12 +11,6 @@ Parameters: ResourceId: Type: String - Subnet01AZ: - Type: String - - Subnet02AZ: - Type: String - AutoMode: Type: String AllowedValues: @@ -49,18 +18,6 @@ Parameters: - "false" Default: "false" -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: "Worker Network Configuration" - Parameters: - - VpcBlock - - PublicSubnet01Block - - PublicSubnet02Block - - PrivateSubnet01Block - - PrivateSubnet02Block - Conditions: HasAdditionalClusterRoleServicePrincipal: Fn::Not: @@ -77,7 +34,7 @@ Resources: VPC: Type: AWS::EC2::VPC Properties: - CidrBlock: !Ref VpcBlock + CidrBlock: {{ .VPCCIDRBlock }} EnableDnsHostnames: true EnableDnsSupport: true Tags: @@ -91,9 +48,6 @@ Resources: VpcId: Ref: VPC - # - # Internet gateways (ipv4, and egress for ipv6) - # InternetGateway: Type: AWS::EC2::InternetGateway Properties: @@ -114,47 +68,29 @@ Resources: VpcId: Ref: VPC - # - # Nat gateways - # - NATGateway01: - Type: AWS::EC2::NatGateway - DependsOn: - - NatGatewayEIP1 - - SubnetPublic01 - - VPCGatewayAttachment - Properties: - AllocationId: - Fn::GetAtt: - - NatGatewayEIP1 - - AllocationId - SubnetId: - Ref: SubnetPublic01 - Tags: - - Key: Name - Value: - Fn::Sub: "${AWS::StackName}/NATGateway01" - NATGateway02: - Type: AWS::EC2::NatGateway - DependsOn: - - NatGatewayEIP2 - - SubnetPublic02 - - VPCGatewayAttachment +{{ range $index, $cidr := .PublicCIDRs }} + SubnetPublic{{ $index }}: + Type: AWS::EC2::Subnet + Metadata: + Comment: Subnet {{ $index }} + DependsOn: IPv6CidrBlock Properties: - AllocationId: - Fn::GetAtt: - - NatGatewayEIP2 - - AllocationId - SubnetId: - Ref: SubnetPublic02 + AvailabilityZone: "{{index $.AvailabilityZones $index}}" + CidrBlock: {{ $cidr }} + Ipv6CidrBlock: + !Select [{{ $index }}, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], {{ $.NumSubnets }}, {{ $.Ipv6CidrBits }}]] + AssignIpv6AddressOnCreation: true + MapPublicIpOnLaunch: true Tags: + - Key: kubernetes.io/role/elb + Value: "1" - Key: Name Value: - Fn::Sub: "${AWS::StackName}/NATGateway02" - # - # Nat Gateway IPs - # - NatGatewayEIP1: + Fn::Sub: "${AWS::StackName}/SubnetPublic{{ $index }}" + VpcId: + Ref: VPC + + NatGatewayEIP{{ $index }}: Type: AWS::EC2::EIP DependsOn: - VPCGatewayAttachment @@ -163,21 +99,35 @@ Resources: Tags: - Key: Name Value: - Fn::Sub: "${AWS::StackName}/NatGatewayEIP1" - NatGatewayEIP2: - Type: AWS::EC2::EIP + Fn::Sub: "${AWS::StackName}/NatGatewayEIP{{ $index }}" + + NATGateway{{ $index }}: + Type: AWS::EC2::NatGateway DependsOn: + - NatGatewayEIP{{ $index }} + - SubnetPublic{{ $index }} - VPCGatewayAttachment Properties: - Domain: vpc + AllocationId: + Fn::GetAtt: + - NatGatewayEIP{{ $index }} + - AllocationId + SubnetId: + Ref: SubnetPublic{{ $index }} Tags: - Key: Name Value: - Fn::Sub: "${AWS::StackName}/NatGatewayEIP2" + Fn::Sub: "${AWS::StackName}/NATGateway{{ $index }}" + + RouteTableAssociationPublic{{ $index }}: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: PublicRouteTable + SubnetId: + Ref: SubnetPublic{{ $index }} +{{ end }} - # - # Routing - public subnets - # PublicRouteTable: Type: AWS::EC2::RouteTable Properties: @@ -210,20 +160,25 @@ Resources: RouteTableId: Ref: PublicRouteTable - # - # Routing - private subnets - # Route tables - # - PrivateRouteTable01: - Type: AWS::EC2::RouteTable +{{ range $index, $cidr := .PrivateCIDRs }} + SubnetPrivate{{ $index }}: + Type: AWS::EC2::Subnet + DependsOn: IPv6CidrBlock Properties: - VpcId: - Ref: VPC + AvailabilityZone: "{{index $.AvailabilityZones $index}}" + CidrBlock: "{{ $cidr }}" + Ipv6CidrBlock: + !Select [{{$indexPadding := len $.PublicCIDRs}}{{ add $indexPadding $index }}, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], {{ $.NumSubnets }}, {{ $.Ipv6CidrBits }}]] + AssignIpv6AddressOnCreation: true Tags: + - Key: kubernetes.io/role/internal-elb + Value: "1" - Key: Name Value: - Fn::Sub: "${AWS::StackName}/PrivateRouteTable01" - PrivateRouteTable02: + Fn::Sub: "${AWS::StackName}/SubnetPrivate{{ $index }}" + VpcId: + Ref: VPC + PrivateRouteTable{{ $index }}: Type: AWS::EC2::RouteTable Properties: VpcId: @@ -231,175 +186,37 @@ Resources: Tags: - Key: Name Value: - Fn::Sub: "${AWS::StackName}/PrivateRouteTable02" - # - # Nat IPv4 Private Routes - # - PrivateSubnetDefaultRoute01: + Fn::Sub: "${AWS::StackName}/PrivateRouteTable{{ $index }}" + + PrivateSubnetDefaultRoute{{ $index }}: Type: AWS::EC2::Route DependsOn: - VPCGatewayAttachment - - NATGateway01 + - NATGateway{{ $index }} Properties: DestinationCidrBlock: 0.0.0.0/0 NatGatewayId: - Ref: NATGateway01 + Ref: NATGateway{{ $index }} RouteTableId: - Ref: PrivateRouteTable01 - PrivateSubnetDefaultRoute02: - Type: AWS::EC2::Route - DependsOn: - - VPCGatewayAttachment - - NATGateway02 - Properties: - DestinationCidrBlock: 0.0.0.0/0 - NatGatewayId: - Ref: NATGateway02 - RouteTableId: - Ref: PrivateRouteTable02 + Ref: PrivateRouteTable{{ $index }} - # - # EOIG IPv6 Private Routes - # - PrivateSubnetDefaultIpv6Route01: + PrivateSubnetDefaultIpv6Route{{ $index }}: Type: AWS::EC2::Route Properties: DestinationIpv6CidrBlock: ::/0 EgressOnlyInternetGatewayId: Ref: EgressOnlyInternetGateway RouteTableId: - Ref: PrivateRouteTable01 - PrivateSubnetDefaultIpv6Route02: - Type: AWS::EC2::Route - Properties: - DestinationIpv6CidrBlock: ::/0 - EgressOnlyInternetGatewayId: - Ref: EgressOnlyInternetGateway - RouteTableId: - Ref: PrivateRouteTable02 + Ref: PrivateRouteTable{{ $index }} - # - # Public subnets - SubnetPublic01: - Type: AWS::EC2::Subnet - Metadata: - Comment: Subnet 01 - DependsOn: IPv6CidrBlock - Properties: - AvailabilityZone: - Ref: Subnet01AZ - CidrBlock: - Ref: PublicSubnet01Block - Ipv6CidrBlock: - !Select [0, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 8, 64]] - AssignIpv6AddressOnCreation: true - MapPublicIpOnLaunch: true - Tags: - - Key: kubernetes.io/role/elb - Value: "1" - - Key: Name - Value: - Fn::Sub: "${AWS::StackName}/SubnetPublic01" - VpcId: - Ref: VPC - SubnetPublic02: - Type: AWS::EC2::Subnet - DependsOn: IPv6CidrBlock - Properties: - AvailabilityZone: - Ref: Subnet02AZ - CidrBlock: - Ref: PublicSubnet02Block - Ipv6CidrBlock: - !Select [1, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 8, 64]] - AssignIpv6AddressOnCreation: true - MapPublicIpOnLaunch: true - Tags: - - Key: kubernetes.io/role/elb - Value: "1" - - Key: Name - Value: - Fn::Sub: "${AWS::StackName}/SubnetPublic02" - VpcId: - Ref: VPC - - # - # Public route table associations - # - RouteTableAssociationPublic01: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - RouteTableId: - Ref: PublicRouteTable - SubnetId: - Ref: SubnetPublic01 - RouteTableAssociationPublic02: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - RouteTableId: - Ref: PublicRouteTable - SubnetId: - Ref: SubnetPublic02 - - # - # Private subnets - # - SubnetPrivate01: - Type: AWS::EC2::Subnet - DependsOn: IPv6CidrBlock - Properties: - AvailabilityZone: - Ref: Subnet01AZ - CidrBlock: - Ref: PrivateSubnet01Block - Ipv6CidrBlock: - !Select [2, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 8, 64]] - AssignIpv6AddressOnCreation: true - Tags: - - Key: kubernetes.io/role/internal-elb - Value: "1" - - Key: Name - Value: - Fn::Sub: "${AWS::StackName}/SubnetPrivate01" - VpcId: - Ref: VPC - SubnetPrivate02: - Type: AWS::EC2::Subnet - DependsOn: IPv6CidrBlock - Properties: - AvailabilityZone: - Ref: Subnet02AZ - CidrBlock: - Ref: PrivateSubnet02Block - Ipv6CidrBlock: - !Select [3, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 8, 64]] - AssignIpv6AddressOnCreation: true - Tags: - - Key: kubernetes.io/role/internal-elb - Value: "1" - - Key: Name - Value: - Fn::Sub: "${AWS::StackName}/SubnetPrivate02" - VpcId: - Ref: VPC - - # - # Private route table associations - # - RouteTableAssociationPrivate01: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - RouteTableId: - Ref: PrivateRouteTable01 - SubnetId: - Ref: SubnetPrivate01 - RouteTableAssociationPrivate02: + RouteTableAssociationPrivate{{ $index }}: Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: - Ref: PrivateRouteTable02 + Ref: PrivateRouteTable{{ $index }} SubnetId: - Ref: SubnetPrivate02 + Ref: SubnetPrivate{{ $index }} +{{ end }} ClusterRole: Type: AWS::IAM::Role @@ -532,8 +349,7 @@ Outputs: Value: Fn::Join: - "," - - - Ref: SubnetPrivate01 - - Ref: SubnetPrivate02 + - [{{ range $index, $cidr := .PrivateCIDRs -}}{{if $index}}, {{end}}Ref: SubnetPrivate{{ $index }}{{ end }}] Export: Name: Fn::Sub: "${AWS::StackName}::SubnetsPrivate" @@ -542,8 +358,7 @@ Outputs: Value: Fn::Join: - "," - - - Ref: SubnetPublic01 - - Ref: SubnetPublic02 + - [{{ range $index, $cidr := .PublicCIDRs -}}{{if $index}}, {{end}}Ref: SubnetPublic{{ $index }}{{ end }}] Export: Name: Fn::Sub: "${AWS::StackName}::SubnetsPublic" @@ -581,4 +396,4 @@ Outputs: - !Ref NodeRole Export: Name: - Fn::Sub: "${AWS::StackName}::NodeRole" + Fn::Sub: "${AWS::StackName}::NodeRole" \ No newline at end of file diff --git a/internal/deployers/eksapi/templates/templates.go b/internal/deployers/eksapi/templates/templates.go index 9837bdf51..bed98c5b2 100644 --- a/internal/deployers/eksapi/templates/templates.go +++ b/internal/deployers/eksapi/templates/templates.go @@ -5,13 +5,21 @@ import ( "text/template" ) -//go:embed infra.yaml -var Infrastructure string - var ( //go:embed unmanaged-nodegroup.yaml.template unmanagedNodegroupTemplate string UnmanagedNodegroup = template.Must(template.New("unmanagedNodegroup").Parse(unmanagedNodegroupTemplate)) + + // for addition in a template + funcMap = template.FuncMap{ + "add": func(val1 int, val2 int) int { + return val1 + val2 + }, + } + + //go:embed infra.yaml.template + infrastructureTemplate string + Infrastructure = template.Must(template.New("infrastructure").Funcs(funcMap).Parse(infrastructureTemplate)) ) type UnmanagedNodegroupTemplateData struct { diff --git a/internal/deployers/eksapi/vpc.go b/internal/deployers/eksapi/vpc.go new file mode 100644 index 000000000..e49d956cb --- /dev/null +++ b/internal/deployers/eksapi/vpc.go @@ -0,0 +1,70 @@ +package eksapi + +import ( + "fmt" + "math" +) + +const ( + baseVPCIPFormat = "192.168.%d.0" + subnetBits = 8 + baseNetMask = 16 +) + +var ( + vpcCidrIP = fmt.Sprintf(baseVPCIPFormat, 0) + vpcCidrBlock = fmt.Sprintf("%s/%d", vpcCidrIP, baseNetMask) +) + +type vpcConfig struct { + VPCCIDRBlock string + PublicCIDRs []string + PrivateCIDRs []string + + // AZs only used for getting AZ by index of a subnet CIDR, doesn't have to be unique + AvailabilityZones []string + Ipv6CidrBits int + NumSubnets int +} + +func getNumSubnetBits(numSubnets int) int { + return int(math.Ceil(math.Log2(float64(numSubnets)))) +} + +func getValidSubnetCIDRs(numSubnets int) (subnets []string, err error) { + if numSubnets > 256 { + // logic gets significantly more messy here and also + // IP range gets extremely restrictive + // error only here for visibility, will probably never + // be needed + return nil, fmt.Errorf("cannot create more than 256 subnets") + } + numNetBitsNeeded := getNumSubnetBits(numSubnets) + subnetNetMask := baseNetMask + numNetBitsNeeded + shift := subnetBits - numNetBitsNeeded + for subnetId := range numSubnets { + subnetBits := subnetId << shift + subnetCidrIP := fmt.Sprintf(baseVPCIPFormat, subnetBits) + subnets = append(subnets, fmt.Sprintf("%s/%d", subnetCidrIP, subnetNetMask)) + } + return subnets, nil +} + +func getVpcConfig(availabilityZones []string) (*vpcConfig, error) { + numAZs := len(availabilityZones) + // keep it simple so the template can get the AZ by index of the CIDR + numSubnets := numAZs * 2 // 1 private, 1 public per AZ + subnetCidrs, err := getValidSubnetCIDRs(numSubnets) + if err != nil { + return nil, err + } + return &vpcConfig{ + VPCCIDRBlock: vpcCidrBlock, + PublicCIDRs: subnetCidrs[:numSubnets/2], // use first half as public + PrivateCIDRs: subnetCidrs[numSubnets/2:], // use second half as private + AvailabilityZones: availabilityZones, + NumSubnets: numSubnets, + // https://docs.aws.amazon.com/vpc/latest/userguide/vpc-cidr-blocks.html#vpc-sizing-ipv6 + Ipv6CidrBits: 64, // leaves 12 subnet bits, ipv4 is the bottleneck + }, nil +} diff --git a/internal/deployers/eksapi/vpc_test.go b/internal/deployers/eksapi/vpc_test.go new file mode 100644 index 000000000..432ab3b22 --- /dev/null +++ b/internal/deployers/eksapi/vpc_test.go @@ -0,0 +1,56 @@ +package eksapi + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +type VPCTestCase struct { + name string + numSubnets int + expectedCIDRs []string + wantErr bool // Optional, for error checking +} + +func Test_getValidSubnetCIDRs(t *testing.T) { + testCases := []struct { + name string + numSubnets int + expectedCIDRs []string + wantErr bool // Optional, for error checking + }{ + { + // more subnets than supported (256) + numSubnets: 257, + wantErr: true, + }, + { + // single subnet claims all IPs + numSubnets: 1, + expectedCIDRs: []string{"192.168.0.0/16"}, + }, + { + numSubnets: 2, + expectedCIDRs: []string{"192.168.0.0/17", "192.168.128.0/17"}, + }, + { + numSubnets: 4, + expectedCIDRs: []string{"192.168.0.0/18", "192.168.64.0/18", "192.168.128.0/18", "192.168.192.0/18"}, + }, + { + // each subnet takes a 1 of the 256 available values in the third octet + numSubnets: 256, + expectedCIDRs: []string{"192.168.0.0/24", "192.168.1.0/24", "192.168.2.0/24", "192.168.3.0/24", "192.168.4.0/24", "192.168.5.0/24", "192.168.6.0/24", "192.168.7.0/24", "192.168.8.0/24", "192.168.9.0/24", "192.168.10.0/24", "192.168.11.0/24", "192.168.12.0/24", "192.168.13.0/24", "192.168.14.0/24", "192.168.15.0/24", "192.168.16.0/24", "192.168.17.0/24", "192.168.18.0/24", "192.168.19.0/24", "192.168.20.0/24", "192.168.21.0/24", "192.168.22.0/24", "192.168.23.0/24", "192.168.24.0/24", "192.168.25.0/24", "192.168.26.0/24", "192.168.27.0/24", "192.168.28.0/24", "192.168.29.0/24", "192.168.30.0/24", "192.168.31.0/24", "192.168.32.0/24", "192.168.33.0/24", "192.168.34.0/24", "192.168.35.0/24", "192.168.36.0/24", "192.168.37.0/24", "192.168.38.0/24", "192.168.39.0/24", "192.168.40.0/24", "192.168.41.0/24", "192.168.42.0/24", "192.168.43.0/24", "192.168.44.0/24", "192.168.45.0/24", "192.168.46.0/24", "192.168.47.0/24", "192.168.48.0/24", "192.168.49.0/24", "192.168.50.0/24", "192.168.51.0/24", "192.168.52.0/24", "192.168.53.0/24", "192.168.54.0/24", "192.168.55.0/24", "192.168.56.0/24", "192.168.57.0/24", "192.168.58.0/24", "192.168.59.0/24", "192.168.60.0/24", "192.168.61.0/24", "192.168.62.0/24", "192.168.63.0/24", "192.168.64.0/24", "192.168.65.0/24", "192.168.66.0/24", "192.168.67.0/24", "192.168.68.0/24", "192.168.69.0/24", "192.168.70.0/24", "192.168.71.0/24", "192.168.72.0/24", "192.168.73.0/24", "192.168.74.0/24", "192.168.75.0/24", "192.168.76.0/24", "192.168.77.0/24", "192.168.78.0/24", "192.168.79.0/24", "192.168.80.0/24", "192.168.81.0/24", "192.168.82.0/24", "192.168.83.0/24", "192.168.84.0/24", "192.168.85.0/24", "192.168.86.0/24", "192.168.87.0/24", "192.168.88.0/24", "192.168.89.0/24", "192.168.90.0/24", "192.168.91.0/24", "192.168.92.0/24", "192.168.93.0/24", "192.168.94.0/24", "192.168.95.0/24", "192.168.96.0/24", "192.168.97.0/24", "192.168.98.0/24", "192.168.99.0/24", "192.168.100.0/24", "192.168.101.0/24", "192.168.102.0/24", "192.168.103.0/24", "192.168.104.0/24", "192.168.105.0/24", "192.168.106.0/24", "192.168.107.0/24", "192.168.108.0/24", "192.168.109.0/24", "192.168.110.0/24", "192.168.111.0/24", "192.168.112.0/24", "192.168.113.0/24", "192.168.114.0/24", "192.168.115.0/24", "192.168.116.0/24", "192.168.117.0/24", "192.168.118.0/24", "192.168.119.0/24", "192.168.120.0/24", "192.168.121.0/24", "192.168.122.0/24", "192.168.123.0/24", "192.168.124.0/24", "192.168.125.0/24", "192.168.126.0/24", "192.168.127.0/24", "192.168.128.0/24", "192.168.129.0/24", "192.168.130.0/24", "192.168.131.0/24", "192.168.132.0/24", "192.168.133.0/24", "192.168.134.0/24", "192.168.135.0/24", "192.168.136.0/24", "192.168.137.0/24", "192.168.138.0/24", "192.168.139.0/24", "192.168.140.0/24", "192.168.141.0/24", "192.168.142.0/24", "192.168.143.0/24", "192.168.144.0/24", "192.168.145.0/24", "192.168.146.0/24", "192.168.147.0/24", "192.168.148.0/24", "192.168.149.0/24", "192.168.150.0/24", "192.168.151.0/24", "192.168.152.0/24", "192.168.153.0/24", "192.168.154.0/24", "192.168.155.0/24", "192.168.156.0/24", "192.168.157.0/24", "192.168.158.0/24", "192.168.159.0/24", "192.168.160.0/24", "192.168.161.0/24", "192.168.162.0/24", "192.168.163.0/24", "192.168.164.0/24", "192.168.165.0/24", "192.168.166.0/24", "192.168.167.0/24", "192.168.168.0/24", "192.168.169.0/24", "192.168.170.0/24", "192.168.171.0/24", "192.168.172.0/24", "192.168.173.0/24", "192.168.174.0/24", "192.168.175.0/24", "192.168.176.0/24", "192.168.177.0/24", "192.168.178.0/24", "192.168.179.0/24", "192.168.180.0/24", "192.168.181.0/24", "192.168.182.0/24", "192.168.183.0/24", "192.168.184.0/24", "192.168.185.0/24", "192.168.186.0/24", "192.168.187.0/24", "192.168.188.0/24", "192.168.189.0/24", "192.168.190.0/24", "192.168.191.0/24", "192.168.192.0/24", "192.168.193.0/24", "192.168.194.0/24", "192.168.195.0/24", "192.168.196.0/24", "192.168.197.0/24", "192.168.198.0/24", "192.168.199.0/24", "192.168.200.0/24", "192.168.201.0/24", "192.168.202.0/24", "192.168.203.0/24", "192.168.204.0/24", "192.168.205.0/24", "192.168.206.0/24", "192.168.207.0/24", "192.168.208.0/24", "192.168.209.0/24", "192.168.210.0/24", "192.168.211.0/24", "192.168.212.0/24", "192.168.213.0/24", "192.168.214.0/24", "192.168.215.0/24", "192.168.216.0/24", "192.168.217.0/24", "192.168.218.0/24", "192.168.219.0/24", "192.168.220.0/24", "192.168.221.0/24", "192.168.222.0/24", "192.168.223.0/24", "192.168.224.0/24", "192.168.225.0/24", "192.168.226.0/24", "192.168.227.0/24", "192.168.228.0/24", "192.168.229.0/24", "192.168.230.0/24", "192.168.231.0/24", "192.168.232.0/24", "192.168.233.0/24", "192.168.234.0/24", "192.168.235.0/24", "192.168.236.0/24", "192.168.237.0/24", "192.168.238.0/24", "192.168.239.0/24", "192.168.240.0/24", "192.168.241.0/24", "192.168.242.0/24", "192.168.243.0/24", "192.168.244.0/24", "192.168.245.0/24", "192.168.246.0/24", "192.168.247.0/24", "192.168.248.0/24", "192.168.249.0/24", "192.168.250.0/24", "192.168.251.0/24", "192.168.252.0/24", "192.168.253.0/24", "192.168.254.0/24", "192.168.255.0/24"}, + }, + } + for _, tc := range testCases { + subnetCidrs, err := getValidSubnetCIDRs(tc.numSubnets) + if !tc.wantErr { + assert.NoError(t, err) + } else { + assert.Error(t, err) + } + assert.Equal(t, tc.expectedCIDRs, subnetCidrs) + } +}