From c53585391af9c049e258a5a214cf458fb8f8956a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 2 Aug 2020 16:12:52 -0700 Subject: [PATCH] headless service + statefulset EnableServiceLinks PublishNotReadyAddresses --- controllers/controller_master.go | 18 ++--- controllers/controller_master_service.go | 14 ++-- controllers/controller_master_statefulset.go | 77 ++++++++++++++++++-- 3 files changed, 87 insertions(+), 22 deletions(-) diff --git a/controllers/controller_master.go b/controllers/controller_master.go index 4a9dc41..6f74183 100644 --- a/controllers/controller_master.go +++ b/controllers/controller_master.go @@ -24,10 +24,8 @@ func (r *SeaweedReconciler) ensureMaster(seaweedCR *seaweedv1.Seaweed) (done boo return done, result, err } - for masterIndex := 0; masterIndex < MasterClusterSize; masterIndex++ { - if done, result, err = r.ensureMasterService(seaweedCR, masterIndex); done { - return done, result, err - } + if done, result, err = r.ensureMasterService(seaweedCR); done { + return done, result, err } return false, ctrl.Result{}, nil @@ -49,7 +47,7 @@ func (r *SeaweedReconciler) ensureMasterStatefulSet(seaweedCR *seaweedv1.Seaweed return true, ctrl.Result{}, err } // Deployment created successfully - return and requeue - return true, ctrl.Result{Requeue: true}, nil + return false, ctrl.Result{}, nil } else if err != nil { log.Error(err, "Failed to get Deployment") return true, ctrl.Result{}, err @@ -58,7 +56,7 @@ func (r *SeaweedReconciler) ensureMasterStatefulSet(seaweedCR *seaweedv1.Seaweed return false, ctrl.Result{}, nil } -func (r *SeaweedReconciler) ensureMasterService(seaweedCR *seaweedv1.Seaweed, masterIndex int) (bool, ctrl.Result, error) { +func (r *SeaweedReconciler) ensureMasterService(seaweedCR *seaweedv1.Seaweed) (bool, ctrl.Result, error) { ctx := context.Background() log := r.Log.WithValues("sw-master-service", seaweedCR.Name) @@ -66,17 +64,17 @@ func (r *SeaweedReconciler) ensureMasterService(seaweedCR *seaweedv1.Seaweed, ma err := r.Get(ctx, types.NamespacedName{Name: seaweedCR.Name, Namespace: seaweedCR.Namespace}, masterService) if err != nil && errors.IsNotFound(err) { // Define a new deployment - dep := r.createMasterService(seaweedCR, masterIndex) + dep := r.createMasterService(seaweedCR) log.Info("Creating a new master headless service", "Namespace", dep.Namespace, "Name", dep.Name) err = r.Create(ctx, dep) if err != nil { - log.Error(err, "Failed to creater service master", "masterIndex", masterIndex) + log.Error(err, "Failed to creater service master", "Namespace", dep.Namespace, "Name", dep.Name) return true, ctrl.Result{}, err } // Deployment created successfully - return and requeue - return true, ctrl.Result{Requeue: true}, nil + return false, ctrl.Result{}, nil } else if err != nil { - log.Error(err, "Failed to get service master", "masterIndex", masterIndex) + log.Error(err, "Failed to get service master", "Namespace", seaweedCR.Namespace, "Name", seaweedCR.Name) return true, ctrl.Result{}, err } log.Info("Get master service " + masterService.Name) diff --git a/controllers/controller_master_service.go b/controllers/controller_master_service.go index 61f5f86..73b9f99 100644 --- a/controllers/controller_master_service.go +++ b/controllers/controller_master_service.go @@ -1,8 +1,6 @@ package controllers import ( - "fmt" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -10,17 +8,21 @@ import ( seaweedv1 "github.com/seaweedfs/seaweedfs-operator/api/v1" ) -func (r *SeaweedReconciler) createMasterService(m *seaweedv1.Seaweed, masterIndex int) *corev1.Service { +func (r *SeaweedReconciler) createMasterService(m *seaweedv1.Seaweed) *corev1.Service { labels := labelsForMaster(m.Name) - labels["name"] = fmt.Sprintf("master-%d", masterIndex) dep := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: m.Name, + Name: m.Name + "-master", Namespace: m.Namespace, + Labels: labels, + Annotations: map[string]string{ + "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", + }, }, Spec: corev1.ServiceSpec{ - ClusterIP: "None", + ClusterIP: "None", + PublishNotReadyAddresses: true, Ports: []corev1.ServicePort{ { Name: "swfs-master", diff --git a/controllers/controller_master_statefulset.go b/controllers/controller_master_statefulset.go index a63aac8..3edb8a2 100644 --- a/controllers/controller_master_statefulset.go +++ b/controllers/controller_master_statefulset.go @@ -13,14 +13,24 @@ import ( func (r *SeaweedReconciler) createMasterStatefulSet(m *seaweedv1.Seaweed) *appsv1.StatefulSet { labels := labelsForMaster(m.Name) replicas := int32(MasterClusterSize) + rollingUpdatePartition := int32(0) + enableServiceLinks := false dep := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: m.Name, + Name: m.Name + "-master", Namespace: m.Namespace, }, Spec: appsv1.StatefulSetSpec{ - Replicas: &replicas, + ServiceName: m.Name + "-master", + PodManagementPolicy: appsv1.ParallelPodManagement, + Replicas: &replicas, + UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{ + Partition: &rollingUpdatePartition, + }, + }, Selector: &metav1.LabelSelector{ MatchLabels: labels, }, @@ -29,9 +39,25 @@ func (r *SeaweedReconciler) createMasterStatefulSet(m *seaweedv1.Seaweed) *appsv Labels: labels, }, Spec: corev1.PodSpec{ + /* + Affinity: &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + */ + EnableServiceLinks: &enableServiceLinks, Containers: []corev1.Container{{ - Image: "chrislusf/seaweedfs:latest", - Name: "master", + Name: "seaweedfs", + Image: "chrislusf/seaweedfs:latest", + ImagePullPolicy: corev1.PullIfNotPresent, Env: []corev1.EnvVar{ { Name: "POD_IP", @@ -61,9 +87,11 @@ func (r *SeaweedReconciler) createMasterStatefulSet(m *seaweedv1.Seaweed) *appsv Command: []string{ "weed", "master", + "-volumePreallocate", "-volumeSizeLimitMB=1000", - "-ip=$(POD_NAME)", - fmt.Sprintf("-peers=%s-0:9333,%s-1:9333,%s-2:9333", m.Name, m.Name, m.Name), + fmt.Sprintf("-ip=$(POD_NAME).%s-master", m.Name), + fmt.Sprintf("-peers=%s-master-0.%s-master:9333,%s-master-1.%s-master:9333,%s-master-2.%s-master:9333", + m.Name, m.Name, m.Name, m.Name, m.Name, m.Name), }, Ports: []corev1.ContainerPort{ { @@ -74,6 +102,43 @@ func (r *SeaweedReconciler) createMasterStatefulSet(m *seaweedv1.Seaweed) *appsv ContainerPort: 19333, }, }, + /* + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/cluster/status", + Port: intstr.IntOrString{ + Type: 0, + IntVal: 9333, + }, + Scheme: "http", + }, + }, + InitialDelaySeconds: 5, + TimeoutSeconds: 0, + PeriodSeconds: 15, + SuccessThreshold: 2, + FailureThreshold: 100, + }, + LivenessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/cluster/status", + Port: intstr.IntOrString{ + Type: 0, + IntVal: 9333, + }, + Scheme: "http", + }, + }, + InitialDelaySeconds: 20, + TimeoutSeconds: 0, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 6, + }, + + */ }}, }, },