Ingress Nginx 落地实践

// Check returns if the nginx healthz endpoint is returning ok (status code 200)
func (n *NGINXController) Check(_ *http.Request) error {
	if n.isShuttingDown {
		return fmt.Errorf("the ingress controller is shutting down")
	}
	// check the nginx master process is running
	fs, err := proc.NewFS("/proc", false)
	if err != nil {
		return errors.Wrap(err, "reading /proc directory")
	}
	f, err := ioutil.ReadFile(nginx.PID)
	if err != nil {
		return errors.Wrapf(err, "reading %v", nginx.PID)
	}
	pid, err := strconv.Atoi(strings.TrimRight(string(f), "\r\n"))
	if err != nil {
		return errors.Wrapf(err, "reading NGINX PID from file %v", nginx.PID)
	}
	_, err = fs.NewProc(pid)
	if err != nil {
		return errors.Wrapf(err, "checking for NGINX process with PID %v", pid)
	}
	statusCode, _, err := nginx.NewGetStatusRequest(nginx.HealthPath)
	if err != nil {
		return errors.Wrapf(err, "checking if NGINX is running")
	}
	if statusCode != 200 {
		return fmt.Errorf("ingress controller is not healthy (%v)", statusCode)
	}
	statusCode, _, err = nginx.NewGetStatusRequest("/is-dynamic-lb-initialized")
	if err != nil {
		return errors.Wrapf(err, "checking if the dynamic load balancer started")
	}
	if statusCode != 200 {
		return fmt.Errorf("dynamic load balancer not started")
	}
	return nil
}

// Start starts a new NGINX master process running in the foreground.
func (n *NGINXController) Start() {
	klog.Info("Starting NGINX Ingress controller")
	n.store.Run(n.stopCh)
	// we need to use the defined ingress class to allow multiple leaders in order to update information about ingress status
	electionID := fmt.Sprintf("%v-%v", n.cfg.ElectionID, class.DefaultClass)
	if class.IngressClass != "" {
		electionID = fmt.Sprintf("%v-%v", n.cfg.ElectionID, class.IngressClass)
	}
	setupLeaderElection(&leaderElectionConfig{
		Client:     n.cfg.Client,
		ElectionID: electionID,
		OnStartedLeading: func(stopCh chan struct{}) {
			if n.syncStatus != nil {
				go n.syncStatus.Run(stopCh)
			}
			n.metricCollector.OnStartedLeading(electionID)
			// manually update SSL expiration metrics (to not wait for a reload)
			n.metricCollector.SetSSLExpireTime(n.runningConfig.Servers)
		},
		OnStoppedLeading: func() {
			n.metricCollector.OnStoppedLeading(electionID)
		},
		PodName:      n.podInfo.Name,
		PodNamespace: n.podInfo.Namespace,
	})
	cmd := n.command.ExecCommand()
	// put NGINX in another process group to prevent it to receive signals meant for the controller
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Setpgid: true,
		Pgid:    0,
	}
	if n.cfg.EnableSSLPassthrough {
		n.setupSSLProxy()
	}
	klog.Info("Starting NGINX process")
	n.start(cmd)
	go n.syncQueue.Run(time.Second, n.stopCh)
	// force initial sync
	n.syncQueue.EnqueueTask(task.GetDummyObject("initial-sync"))
	// In case of error the temporal configuration file will be available up to five minutes after the error
	go func() {
		for {
			time.Sleep(5 * time.Minute)
			err := cleanTempNginxCfg()
			if err != nil {
				klog.Infof("Unexpected error removing temporal configuration files: %v", err)
			}
		}
	}()
	if n.validationWebhookServer != nil {
		klog.Infof("Starting validation webhook on %s with keys %s %s", n.validationWebhookServer.Addr, n.cfg.ValidationWebhookCertPath, n.cfg.ValidationWebhookKeyPath)
		go func() {
			klog.Error(n.validationWebhookServer.ListenAndServeTLS("", ""))
		}()
	}
	for {
		select {
		case err := <-n.ngxErrCh:
			if n.isShuttingDown {
				break
			}
			// if the nginx master process dies the workers continue to process requests, passing checks but in case of updates in ingress no updates will be reflected in the nginx configuration which can lead to confusion and report issues because of this behavior.
			// To avoid this issue we restart nginx in case of errors.
			if process.IsRespawnIfRequired(err) {
				process.WaitUntilPortIsAvailable(n.cfg.ListenPorts.HTTP)
				// release command resources
				cmd.Process.Release()
				// start a new nginx master process if the controller is not being stopped
				cmd = n.command.ExecCommand()
				cmd.SysProcAttr = &syscall.SysProcAttr{
					Setpgid: true,
					Pgid:    0,
				}
				n.start(cmd)
			}
		case event := <-n.updateCh.Out():
			if n.isShuttingDown {
				break
			}
			if evt, ok := event.(store.Event); ok {
				klog.V(3).Infof("Event %v received - object %v", evt.Type, evt.Obj)
				if evt.Type == store.ConfigurationEvent {
					// TODO: is this necessary? Consider removing this special case
					n.syncQueue.EnqueueTask(task.GetDummyObject("configmap-change"))
					continue
				}
				n.syncQueue.EnqueueSkippableTask(evt.Obj)
			} else {
				klog.Warningf("Unexpected event type received %T", event)
			}
		case <-n.stopCh:
			break
		}
	}
}

livenessProbe:
  failureThreshold: 3
  httpGet:
    path: /healthz
    port: 10254
    scheme: HTTP
  initialDelaySeconds: 10
  periodSeconds: 10
  successThreshold: 1
  timeoutSeconds: 10
readinessProbe:
  failureThreshold: 3
  httpGet:
    path: /healthz
    port: 10254
    scheme: HTTP
  periodSeconds: 10
  successThreshold: 1
  timeoutSeconds: 10

// Stop gracefully stops the NGINX master process.
func (n *NGINXController) Stop() error {
	n.isShuttingDown = true
	n.stopLock.Lock()
	defer n.stopLock.Unlock()
	if n.syncQueue.IsShuttingDown() {
		return fmt.Errorf("shutdown already in progress")
	}
	klog.Info("Shutting down controller queues")
	close(n.stopCh)
	go n.syncQueue.Shutdown()
	if n.syncStatus != nil {
		n.syncStatus.Shutdown()
	}
	if n.validationWebhookServer != nil {
		klog.Info("Stopping admission controller")
		err := n.validationWebhookServer.Close()
		if err != nil {
			return err
		}
	}
	var (
		period int
		perr   error
	)
	periodEnv := os.Getenv("GRACE_STOP_PERIOD")
	if period, perr = strconv.Atoi(periodEnv); perr != nil {
		period = 30
	}
	klog.Infof("Graceful waiting %d second to stop", period)
	time.Sleep(time.Second * time.Duration(period))
	// send stop signal to NGINX
	klog.Info("Stopping NGINX process")
	cmd := n.command.ExecCommand("-s", "quit")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Run()
	if err != nil {
		return err
	}
	// wait for the NGINX process to terminate
	timer := time.NewTicker(time.Second * 1)
	for range timer.C {
		if !nginx.IsRunning() {
			klog.Info("NGINX process has stopped")
			timer.Stop()
			break
		}
	}
	return nil
}

lifecycle:
  preStop:
    exec:
      command:
        - /wait-shutdown

// Stop gracefully stops the NGINX master process.
func (n *NGINXController) Stop() error {
	...
	periodEnv := os.Getenv("GRACE_STOP_PERIOD")
	if period, perr = strconv.Atoi(periodEnv); perr != nil {
		period = 30
	}
	klog.Infof("Graceful waiting %d second to stop", period)
	time.Sleep(time.Second * time.Duration(period))
	...
}

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: ingress-nginx-redefine
spec:
  template:
    metadata:
      annotations:
        prometheus.io/port: '10254'
        prometheus.io/scrape: 'true'
        nginx.ingress.kubernetes.io/force-ssl-redirect: 'false'
    spec:
      serviceAccountName: ingress-nginx-redefine-serviceaccount
      hostNetwork: true
      terminationGracePeriodSeconds: 300
      containers:
        - name: nginx-ingress-controller
          image: nginx-ingress-controller:0.26.1
          args:
            - /nginx-ingress-controller
            - '--configmap=default/ingress-nginx-redefine-configuration'
            - '--default-backend-service=default/ingress-default-http-backend'
            - '--annotations-prefix=nginx.ingress.kubernetes.io'
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: GRACE_STOP_PERIOD
              value: '30'
          ports:
            - name: http
              containerPort: 80
              hostPort: 80
            - name: https
              containerPort: 443
              hostPort: 443
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          lifecycle:
            preStop:
              exec:
                command:
                  - /wait-shutdown
          resources:
            limits:
              cpu: 0
              memory: 0
            requests: {}
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: node-role.kubernetes.io/ingress
                    operator: Exists
      tolerations:
        - operator: Exists