Posted on 

cloud provider summary

​ cloud controller manager 是可插拔的,它运行新的 cloud provider 简单方便的与 Kubernetes 集成。

​ cloud provider 启动从 new 一个 cloud manager command 开始

1
2
3
4
5
func main() {
...
command := app.NewCloudControllerManagerCommand()
...
}

​ Run 函数是 k8s contrller manager 定义的关键入口,在 NewCloudControllerManagerCommand 中被调用,参数是一个 CompletedConfig,第二个参数是 stopCh。

1
2
3
4
5
6
7
8
// NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
func NewCloudControllerManagerCommand() *cobra.Command {
s, err := options.NewCloudControllerManagerOptions()
...
if err := Run(c.Complete(), wait.NeverStop); err != nil {
...
return cmd
}

​ Run 函数的实现,主要是做些 controller 启动前的准备工作,比如锁的获取,实际做 controller 启动的是调用 startControllers。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

// Run runs the ExternalCMServer. This should never exit.
func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error {
...
run := func(ctx context.Context) {
if err := startControllers(c, ctx.Done(), cloud); err != nil {
glog.Fatalf("error running controllers: %v", err)
}
}
...
// Lock required for leader election
rl, err := resourcelock.New(c.ComponentConfig.Generic.LeaderElection.ResourceLock,
"kube-system",
"cloud-controller-manager",
c.LeaderElectionClient.CoreV1(),
resourcelock.ResourceLockConfig{
Identity: id,
EventRecorder: c.EventRecorder,
})
if err != nil {
glog.Fatalf("error creating lock: %v", err)
}
....
}

在 startcontroller 中实际启动的 controller 如下:CloudNodeController,PersistentVolumeLabelController,RouteController,serviceController。

CloudNodeController 负责初始化 k8s 中 node 与云上的信息。

PersistentVolumeLabelController 负责给 PV 打 Label,保证存储不会被跨区挂载。

RouteController 用来创建路由解决不通 node 上 pod 的互通问题。

serviceController 用来创建云厂商的负载均衡。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

// startControllers starts the cloud specific controller loops.
func startControllers(c *cloudcontrollerconfig.CompletedConfig, stop <-chan struct{}, cloud cloudprovider.Interface) error {
...
if cloud != nil {
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(c.ClientBuilder)
}
// Start the CloudNodeController
nodeController := cloudcontrollers.NewCloudNodeController(
c.SharedInformers.Core().V1().Nodes(),
client("cloud-node-controller"), cloud,
c.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration,
c.ComponentConfig.NodeStatusUpdateFrequency.Duration)

nodeController.Run(stop)
...
// Start the PersistentVolumeLabelController
pvlController := cloudcontrollers.NewPersistentVolumeLabelController(client("pvl-controller"), cloud)
go pvlController.Run(5, stop)
...
// Start the service controller
serviceController, err := servicecontroller.New(
cloud,
client("service-controller"),
c.SharedInformers.Core().V1().Services(),
c.SharedInformers.Core().V1().Nodes(),
c.ComponentConfig.KubeCloudShared.ClusterName,
)
if err != nil {
glog.Errorf("Failed to start service controller: %v", err)
} else {
go serviceController.Run(stop, int(c.ComponentConfig.ServiceController.ConcurrentServiceSyncs))
...

// If CIDRs should be allocated for pods and set on the CloudProvider, then start the route controller
if c.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs && c.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes {
if routes, ok := cloud.Routes(); !ok {
glog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.")
} else {
var clusterCIDR *net.IPNet
if len(strings.TrimSpace(c.ComponentConfig.KubeCloudShared.ClusterCIDR)) != 0 {
_, clusterCIDR, err = net.ParseCIDR(c.ComponentConfig.KubeCloudShared.ClusterCIDR)
if err != nil {
glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", c.ComponentConfig.KubeCloudShared.ClusterCIDR, err)
}
}

routeController := routecontroller.New(routes, client("route-controller"), c.SharedInformers.Core().V1().Nodes(), c.ComponentConfig.KubeCloudShared.ClusterName, clusterCIDR)
go routeController.Run(stop, c.ComponentConfig.KubeCloudShared.RouteReconciliationPeriod.Duration)
...
}