Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce a new lock for L4 and related controllers. #2805

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Introduce a new lock for L4 and related controllers. #2805

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 61 additions & 31 deletions cmd/glbc/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ import (
)

const negLockName = "ingress-gce-neg-lock"
const l4LockName = "l4-lb-controller-gce-lock"

func main() {
flags.Register()
Expand Down Expand Up @@ -316,7 +317,7 @@ func main() {
"InstanceGroup controller", flags.F.EnableIGController,
"PSC controller", flags.F.EnablePSC,
)
runControllers(ctx, systemHealth, rOption, logger)
runControllers(ctx, systemHealth, rOption, leOption, logger)
}

if flags.F.LeaderElection.LeaderElect {
Expand Down Expand Up @@ -408,7 +409,7 @@ func makeIngressRunnerWithLeaderElection(
leOption,
flags.F.LeaderElection.LockObjectName,
func(context.Context) {
runControllers(ctx, systemHealth, runOption, logger)
runControllers(ctx, systemHealth, runOption, leOption, logger)
},
func() {
logger.Info("lost master")
Expand Down Expand Up @@ -449,7 +450,7 @@ func makeRunnerWithLeaderElection(
}, nil
}

func runControllers(ctx *ingctx.ControllerContext, systemHealth *systemhealth.SystemHealth, option runOption, logger klog.Logger) {
func runControllers(ctx *ingctx.ControllerContext, systemHealth *systemhealth.SystemHealth, option runOption, leOption leaderElectionOption, logger klog.Logger) {
if flags.F.RunIngressController {
lbc := controller.NewLoadBalancerController(ctx, option.stopCh, logger)
systemHealth.AddHealthCheck("ingress", lbc.SystemHealth)
Expand All @@ -464,42 +465,71 @@ func runControllers(ctx *ingctx.ControllerContext, systemHealth *systemhealth.Sy
logger.V(0).Info("firewall controller started")
}

if flags.F.RunL4Controller {
l4Controller := l4lb.NewILBController(ctx, option.stopCh, logger)
systemHealth.AddHealthCheck(l4lb.L4ILBControllerName, l4Controller.SystemHealth)
runWithWg(l4Controller.Run, option.wg)
logger.V(0).Info("L4 controller started")
}
runL4Controllers(ctx, systemHealth, option, leOption, logger)

if flags.F.EnablePSC {
pscController := psc.NewController(ctx, option.stopCh, logger)
runWithWg(pscController.Run, option.wg)
logger.V(0).Info("PSC Controller started")
ctx.Start(option.stopCh)
}

func runL4Controllers(ctx *ingctx.ControllerContext, systemHealth *systemhealth.SystemHealth, option runOption, leOption leaderElectionOption, logger klog.Logger) {
if !flags.F.RunL4Controller && !flags.F.EnablePSC && !flags.F.EnableIGController && !flags.F.RunL4NetLBController {
return
}
run := func() {
if flags.F.RunL4Controller {
l4Controller := l4lb.NewILBController(ctx, option.stopCh, logger)
systemHealth.AddHealthCheck(l4lb.L4ILBControllerName, l4Controller.SystemHealth)
runWithWg(l4Controller.Run, option.wg)
logger.V(0).Info("L4 controller started")
}

ctx.Start(option.stopCh)
if flags.F.EnablePSC {
pscController := psc.NewController(ctx, option.stopCh, logger)
runWithWg(pscController.Run, option.wg)
logger.V(0).Info("PSC Controller started")
}

if flags.F.EnableIGController {
igControllerParams := &instancegroups.ControllerConfig{
NodeInformer: ctx.NodeInformer,
ZoneGetter: ctx.ZoneGetter,
IGManager: ctx.InstancePool,
HasSynced: ctx.HasSynced,
EnableMultiSubnetCluster: flags.F.EnableIGMultiSubnetCluster,
StopCh: option.stopCh,
if flags.F.EnableIGController {
igControllerParams := &instancegroups.ControllerConfig{
NodeInformer: ctx.NodeInformer,
ZoneGetter: ctx.ZoneGetter,
IGManager: ctx.InstancePool,
HasSynced: ctx.HasSynced,
EnableMultiSubnetCluster: flags.F.EnableIGMultiSubnetCluster,
StopCh: option.stopCh,
}
igController := instancegroups.NewController(igControllerParams, logger)
runWithWg(igController.Run, option.wg)
}
igController := instancegroups.NewController(igControllerParams, logger)
runWithWg(igController.Run, option.wg)
}

// The L4NetLbController will be run when RbsMode flag is Set
if flags.F.RunL4NetLBController {
l4netlbController := l4lb.NewL4NetLBController(ctx, option.stopCh, logger)
systemHealth.AddHealthCheck(l4lb.L4NetLBControllerName, l4netlbController.SystemHealth)
// The L4NetLbController will be run when RbsMode flag is Set
if flags.F.RunL4NetLBController {
l4netlbController := l4lb.NewL4NetLBController(ctx, option.stopCh, logger)
systemHealth.AddHealthCheck(l4lb.L4NetLBControllerName, l4netlbController.SystemHealth)

runWithWg(l4netlbController.Run, option.wg)
logger.V(0).Info("L4NetLB controller started")
runWithWg(l4netlbController.Run, option.wg)
logger.V(0).Info("L4NetLB controller started")
}
}
if !flags.F.LeaderElection.LeaderElect || !flags.F.GateL4ByLock {
run()
return
}
lockLogger := logger.WithValues("lock", l4LockName)
runner, err := makeRunnerWithLeaderElection(leOption, l4LockName, func(ctx context.Context) {
lockLogger.V(0).Info("Acquired L4 Leader election lock")
go collectLockAvailabilityMetrics(l4LockName, flags.F.GKEClusterType, option.stopCh, lockLogger)
run()
}, func() {
lockLogger.V(0).Info("Stop running L4 Leader election")
})
if err != nil {
klog.Fatalf("L4 makeLeaderElectionConfig()=%v, want nil", err)
}
// run in a separate goroutine to not block further operation if lock can't be acquired.
go func() {
lockLogger.V(0).Info("Attempt to acquire L4 Leader election lock")
leaderelection.RunOrDie(context.Background(), *runner)
}()
}

func runNEGController(ctx *ingctx.ControllerContext, systemHealth *systemhealth.SystemHealth, option runOption, logger klog.Logger) {
Expand Down
2 changes: 2 additions & 0 deletions pkg/flags/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ var F = struct {
EnableL4NetLBNEG bool
EnableL4NetLBNEGDefault bool
GateNEGByLock bool
GateL4ByLock bool
EnableMultipleIGs bool
EnableL4StrongSessionAffinity bool
EnableNEGLabelPropagation bool
Expand Down Expand Up @@ -288,6 +289,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5
flag.BoolVar(&F.EnableL4NetLBNEG, "enable-l4-netlb-neg", false, `Optional, if enabled then the NetLB controller can create L4 NetLB services with NEG backends.`)
flag.BoolVar(&F.EnableL4NetLBNEGDefault, "enable-l4-netlb-neg-default", false, `Optional, if enabled then newly created L4 NetLB services will use NEG backends. Has effect only if '--enable-l4-netlb-neg' is set to true.`)
flag.BoolVar(&F.GateNEGByLock, "gate-neg-by-lock", false, "If enabled then the NEG controller will be run via leader election with NEG resource lock")
flag.BoolVar(&F.GateL4ByLock, "gate-l4-by-lock", false, "If enabled then the L4 controllers will be run via leader election with L4 resource lock")
flag.BoolVar(&F.EnableIGController, "enable-ig-controller", true, `Optional, if enabled then the IG controller will be run.`)
flag.BoolVar(&F.EnablePSC, "enable-psc", false, "Enable PSC controller")
flag.StringVar(&F.GKEClusterName, "gke-cluster-name", "", "The name of the GKE cluster this Ingress Controller will be interacting with")
Expand Down