headless service + statefulset

EnableServiceLinks
PublishNotReadyAddresses

controllers/controller_master.go

@@ -24,10 +24,8 @@ func (r *SeaweedReconciler) ensureMaster(seaweedCR *seaweedv1.Seaweed) (done boo
 		return done, result, err
 	}
 
-	for masterIndex := 0; masterIndex < MasterClusterSize; masterIndex++ {
+	if done, result, err = r.ensureMasterService(seaweedCR); done {
-		if done, result, err = r.ensureMasterService(seaweedCR, masterIndex); done {
+		return done, result, err
-			return done, result, err
-		}
 	}
 
 	return false, ctrl.Result{}, nil
@@ -49,7 +47,7 @@ func (r *SeaweedReconciler) ensureMasterStatefulSet(seaweedCR *seaweedv1.Seaweed
 			return true, ctrl.Result{}, err
 		}
 		// Deployment created successfully - return and requeue
-		return true, ctrl.Result{Requeue: true}, nil
+		return false, ctrl.Result{}, nil
 	} else if err != nil {
 		log.Error(err, "Failed to get Deployment")
 		return true, ctrl.Result{}, err
@@ -58,7 +56,7 @@ func (r *SeaweedReconciler) ensureMasterStatefulSet(seaweedCR *seaweedv1.Seaweed
 	return false, ctrl.Result{}, nil
 }
 
-func (r *SeaweedReconciler) ensureMasterService(seaweedCR *seaweedv1.Seaweed, masterIndex int) (bool, ctrl.Result, error) {
+func (r *SeaweedReconciler) ensureMasterService(seaweedCR *seaweedv1.Seaweed) (bool, ctrl.Result, error) {
 	ctx := context.Background()
 	log := r.Log.WithValues("sw-master-service", seaweedCR.Name)
 
@@ -66,17 +64,17 @@ func (r *SeaweedReconciler) ensureMasterService(seaweedCR *seaweedv1.Seaweed, ma
 	err := r.Get(ctx, types.NamespacedName{Name: seaweedCR.Name, Namespace: seaweedCR.Namespace}, masterService)
 	if err != nil && errors.IsNotFound(err) {
 		// Define a new deployment
-		dep := r.createMasterService(seaweedCR, masterIndex)
+		dep := r.createMasterService(seaweedCR)
 		log.Info("Creating a new master headless service", "Namespace", dep.Namespace, "Name", dep.Name)
 		err = r.Create(ctx, dep)
 		if err != nil {
-			log.Error(err, "Failed to creater service master", "masterIndex", masterIndex)
+			log.Error(err, "Failed to creater service master", "Namespace", dep.Namespace, "Name", dep.Name)
 			return true, ctrl.Result{}, err
 		}
 		// Deployment created successfully - return and requeue
-		return true, ctrl.Result{Requeue: true}, nil
+		return false, ctrl.Result{}, nil
 	} else if err != nil {
-		log.Error(err, "Failed to get service master", "masterIndex", masterIndex)
+		log.Error(err, "Failed to get service master", "Namespace", seaweedCR.Namespace, "Name", seaweedCR.Name)
 		return true, ctrl.Result{}, err
 	}
 	log.Info("Get master service " + masterService.Name)

controllers/controller_master_service.go

@@ -1,8 +1,6 @@
 package controllers
 
 import (
-	"fmt"
-
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -10,17 +8,21 @@ import (
 	seaweedv1 "github.com/seaweedfs/seaweedfs-operator/api/v1"
 )
 
-func (r *SeaweedReconciler) createMasterService(m *seaweedv1.Seaweed, masterIndex int) *corev1.Service {
+func (r *SeaweedReconciler) createMasterService(m *seaweedv1.Seaweed) *corev1.Service {
 	labels := labelsForMaster(m.Name)
-	labels["name"] = fmt.Sprintf("master-%d", masterIndex)
 
 	dep := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      m.Name,
+			Name:      m.Name + "-master",
 			Namespace: m.Namespace,
+			Labels:    labels,
+			Annotations: map[string]string{
+				"service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
+			},
 		},
 		Spec: corev1.ServiceSpec{
-			ClusterIP: "None",
+			ClusterIP:                "None",
+			PublishNotReadyAddresses: true,
 			Ports: []corev1.ServicePort{
 				{
 					Name:     "swfs-master",

controllers/controller_master_statefulset.go

@@ -13,14 +13,24 @@ import (
 func (r *SeaweedReconciler) createMasterStatefulSet(m *seaweedv1.Seaweed) *appsv1.StatefulSet {
 	labels := labelsForMaster(m.Name)
 	replicas := int32(MasterClusterSize)
+	rollingUpdatePartition := int32(0)
+	enableServiceLinks := false
 
 	dep := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      m.Name,
+			Name:      m.Name + "-master",
 			Namespace: m.Namespace,
 		},
 		Spec: appsv1.StatefulSetSpec{
-			Replicas: &replicas,
+			ServiceName:         m.Name + "-master",
+			PodManagementPolicy: appsv1.ParallelPodManagement,
+			Replicas:            &replicas,
+			UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
+				Type: appsv1.RollingUpdateStatefulSetStrategyType,
+				RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
+					Partition: &rollingUpdatePartition,
+				},
+			},
 			Selector: &metav1.LabelSelector{
 				MatchLabels: labels,
 			},
@@ -29,9 +39,25 @@ func (r *SeaweedReconciler) createMasterStatefulSet(m *seaweedv1.Seaweed) *appsv
 					Labels: labels,
 				},
 				Spec: corev1.PodSpec{
+					/*
+						Affinity: &corev1.Affinity{
+							PodAntiAffinity: &corev1.PodAntiAffinity{
+								RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
+									{
+										LabelSelector: &metav1.LabelSelector{
+											MatchLabels: labels,
+										},
+										TopologyKey: "kubernetes.io/hostname",
+									},
+								},
+							},
+						},
+					*/
+					EnableServiceLinks: &enableServiceLinks,
 					Containers: []corev1.Container{{
-						Image: "chrislusf/seaweedfs:latest",
+						Name:            "seaweedfs",
-						Name:  "master",
+						Image:           "chrislusf/seaweedfs:latest",
+						ImagePullPolicy: corev1.PullIfNotPresent,
 						Env: []corev1.EnvVar{
 							{
 								Name: "POD_IP",
@@ -61,9 +87,11 @@ func (r *SeaweedReconciler) createMasterStatefulSet(m *seaweedv1.Seaweed) *appsv
 						Command: []string{
 							"weed",
 							"master",
+							"-volumePreallocate",
 							"-volumeSizeLimitMB=1000",
-							"-ip=$(POD_NAME)",
+							fmt.Sprintf("-ip=$(POD_NAME).%s-master", m.Name),
-							fmt.Sprintf("-peers=%s-0:9333,%s-1:9333,%s-2:9333", m.Name, m.Name, m.Name),
+							fmt.Sprintf("-peers=%s-master-0.%s-master:9333,%s-master-1.%s-master:9333,%s-master-2.%s-master:9333",
+								m.Name, m.Name, m.Name, m.Name, m.Name, m.Name),
 						},
 						Ports: []corev1.ContainerPort{
 							{
@@ -74,6 +102,43 @@ func (r *SeaweedReconciler) createMasterStatefulSet(m *seaweedv1.Seaweed) *appsv
 								ContainerPort: 19333,
 							},
 						},
+						/*
+							ReadinessProbe: &corev1.Probe{
+								Handler: corev1.Handler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/cluster/status",
+										Port: intstr.IntOrString{
+											Type:   0,
+											IntVal: 9333,
+										},
+										Scheme: "http",
+									},
+								},
+								InitialDelaySeconds: 5,
+								TimeoutSeconds:      0,
+								PeriodSeconds:       15,
+								SuccessThreshold:    2,
+								FailureThreshold:    100,
+							},
+							LivenessProbe: &corev1.Probe{
+								Handler: corev1.Handler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/cluster/status",
+										Port: intstr.IntOrString{
+											Type:   0,
+											IntVal: 9333,
+										},
+										Scheme: "http",
+									},
+								},
+								InitialDelaySeconds: 20,
+								TimeoutSeconds:      0,
+								PeriodSeconds:       10,
+								SuccessThreshold:    1,
+								FailureThreshold:    6,
+							},
+
+						*/
 					}},
 				},
 			},