Kubernetes官方的Ingress控制器在0.25.0以前是基于Nginx，0.25.0以后基于OpenResty实现。它的原理就是controller通过Watch API Server 监测Ingress资源对象的变化生成对应的Nginx配置文件，然后reload重新加载配置生效，更详细的信息可以参见官方文档。

因为所有配置都放在nginx.conf一个文件，当Ingress多了以后，生成的配置文件会特别长，调试起来很不方便，特别是自0.14.0以后，upstream使用lua-resty-balancer动态生成，调试就更加困难了，所以社区有人提了一个PR增加一个kubectl插件ingress-nginx来方便调试ingress-nginx配置。

安装 ingress-nginx kubectl插件

执行kubectl krew install ingress-nginx 安装插件，需要依赖krew。

关于krew安装可以参见kubectl插件管理工具krew。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

~ kubectl krew install ingress-nginx
Updated the local copy of plugin index.
Installing plugin: ingress-nginx
Installed plugin: ingress-nginx
\
 | Use this plugin:
 | 	kubectl ingress-nginx
 | Documentation:
 | 	https://kubernetes.github.io/ingress-nginx/kubectl-plugin/
/
WARNING: You installed a plugin from the krew-index plugin repository.
   These plugins are not audited for security by the Krew maintainers.
   Run them at your own risk.

使用插件调试ingress-nginx配置

查看upstream后端Real server信息

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

➜  ~ kubectl ingress-nginx backends -n ingress-nginx
[
  {
    "name": "default-nginx-test-pvc-80",
    "service": {
      "metadata": {
        "creationTimestamp": null
      },
      "spec": {
        "ports": [
          {
            "protocol": "TCP",
            "port": 80,
            "targetPort": 80
          }
        ],
        "selector": {
          "run": "nginx-test-pvc"
        },
        "clusterIP": "10.254.204.159",
        "type": "ClusterIP",
        "sessionAffinity": "None"
      },
      "status": {
        "loadBalancer": {}
      }
    },
    "port": 80,
    "secureCACert": {
      "secret": "",
      "caFilename": "",
      "caSha": "",
      "crlFileName": "",
      "crlSha": ""
    },
    "sslPassthrough": false,
    "endpoints": [
      {
        "address": "10.1.25.133",
        "port": "80"
      }
    ],
    "sessionAffinityConfig": {
      "name": "",
      "mode": "",
      "cookieSessionAffinity": {
        "name": ""
      }
    },
    "upstreamHashByConfig": {
      "upstream-hash-by-subset-size": 3
    },
    "noServer": false,
    "trafficShapingPolicy": {
      "weight": 0,
      "header": "",
      "headerValue": "",
      "cookie": ""
    }
  },
  {
    "name": "upstream-default-backend",
    "port": 0,
    "secureCACert": {
      "secret": "",
      "caFilename": "",
      "caSha": "",
      "crlFileName": "",
      "crlSha": ""
    },
    "sslPassthrough": false,
    "endpoints": [
      {
        "address": "127.0.0.1",
        "port": "8181"
      }
    ],
    "sessionAffinityConfig": {
      "name": "",
      "mode": "",
      "cookieSessionAffinity": {
        "name": ""
      }
    },
    "upstreamHashByConfig": {},
    "noServer": false,
    "trafficShapingPolicy": {
      "weight": 0,
      "header": "",
      "headerValue": "",
      "cookie": ""
    }
  }
]

查看nginx配置文件

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

➜  ~ kubectl ingress-nginx conf -n ingress-nginx

# Configuration checksum: 2771639040602310181

# setup custom paths that do not require root access
pid /tmp/nginx.pid;

daemon off;

worker_processes 16;

worker_rlimit_nofile 64512;

worker_shutdown_timeout 240s ;

events {
	multi_accept        on;
	worker_connections  16384;
	use                 epoll;
}

http {
	lua_package_path "/etc/nginx/lua/?.lua;;";

	lua_shared_dict balancer_ewma 10M;
	lua_shared_dict balancer_ewma_last_touched_at 10M;
	lua_shared_dict balancer_ewma_locks 1M;
	lua_shared_dict certificate_data 20M;
	lua_shared_dict certificate_servers 5M;
	lua_shared_dict configuration_data 20M;
...
#太长，省略

完整参数

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

➜ ~ kubectl ingress-nginx -h
A kubectl plugin for inspecting your ingress-nginx deployments

Usage:
  ingress-nginx [command]

Available Commands:
  backends    Inspect the dynamic backend information of an ingress-nginx instance
  certs       Output the certificate data stored in an ingress-nginx pod
  conf        Inspect the generated nginx.conf
  exec        Execute a command inside an ingress-nginx pod
  general     Inspect the other dynamic ingress-nginx information
  help        Help about any command
  info        Show information about the ingress-nginx service
  ingresses   Provide a short summary of all of the ingress definitions
  lint        Inspect kubernetes resources for possible issues
  logs        Get the kubernetes logs for an ingress-nginx pod
  ssh         ssh into a running ingress-nginx pod

Flags:
      --as string                      Username to impersonate for the operation
      --as-group stringArray           Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
      --cache-dir string               Default HTTP cache directory (default "/Users/xnile/.kube/http-cache")
      --certificate-authority string   Path to a cert file for the certificate authority
      --client-certificate string      Path to a client certificate file for TLS
      --client-key string              Path to a client key file for TLS
      --cluster string                 The name of the kubeconfig cluster to use
      --context string                 The name of the kubeconfig context to use
  -h, --help                           help for ingress-nginx
      --insecure-skip-tls-verify       If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure
      --kubeconfig string              Path to the kubeconfig file to use for CLI requests.
  -n, --namespace string               If present, the namespace scope for this CLI request
      --request-timeout string         The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0")
  -s, --server string                  The address and port of the Kubernetes API server
      --token string                   Bearer token for authentication to the API server
      --user string                    The name of the kubeconfig user to use

Use "ingress-nginx [command] --help" for more information about a command.

插件实现原理

ingress-nginx插件其实是靠调用ingress-nginx-controller容器里集成的dbg命令实现获取后端Real Server的，它的代码在这里。

kubectl ingress-nginx backends -n ingress-nginx 命令实际等价于执行了kubectl -n ingress-nginx exec nginx-ingress-controller-799dbf6fbd-w77mn -- /dbg backends all

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

➜  ~ kubectl -n ingress-nginx exec nginx-ingress-controller-799dbf6fbd-w77mn -- /dbg backends all
[
  {
    "name": "default-nginx-test-pvc-80",
    "service": {
      "metadata": {
        "creationTimestamp": null
      },
      "spec": {
        "ports": [
          {
            "protocol": "TCP",
            "port": 80,
            "targetPort": 80
          }
        ],
        "selector": {
          "run": "nginx-test-pvc"
        },
        "clusterIP": "10.254.204.159",
        "type": "ClusterIP",
        "sessionAffinity": "None"
      },
      "status": {
        "loadBalancer": {}
      }
    },
    "port": 80,
    "secureCACert": {
      "secret": "",
      "caFilename": "",
      "caSha": "",
      "crlFileName": "",
      "crlSha": ""
    },
    "sslPassthrough": false,
    "endpoints": [
      {
        "address": "10.1.25.133",
        "port": "80"
      }
    ],
    "sessionAffinityConfig": {
      "name": "",
      "mode": "",
      "cookieSessionAffinity": {
        "name": ""
      }
    },
    "upstreamHashByConfig": {
      "upstream-hash-by-subset-size": 3
    },
    "noServer": false,
    "trafficShapingPolicy": {
      "weight": 0,
      "header": "",
      "headerValue": "",
      "cookie": ""
    }
  },
  {
    "name": "upstream-default-backend",
    "port": 0,
    "secureCACert": {
      "secret": "",
      "caFilename": "",
      "caSha": "",
      "crlFileName": "",
      "crlSha": ""
    },
    "sslPassthrough": false,
    "endpoints": [
      {
        "address": "127.0.0.1",
        "port": "8181"
      }
    ],
    "sessionAffinityConfig": {
      "name": "",
      "mode": "",
      "cookieSessionAffinity": {
        "name": ""
      }
    },
    "upstreamHashByConfig": {},
    "noServer": false,
    "trafficShapingPolicy": {
      "weight": 0,
      "header": "",
      "headerValue": "",
      "cookie": ""
    }
  }
]

同样也可以获取到upstream的后端Real server信息。

注意

因为前边已经分析了插件其实是依靠nginx-ingress-controller里的集成的dbg命令实现的，所以要想正常使用插件，nginx-ingress-controller的版本必须大于0.23.0,否则执行命令时会提示下边错误。

1
2

➜  ~ kubectl ingress-nginx backends -n kube-system
exit status 126

参考

https://kubernetes.github.io/ingress-nginx/how-it-works/

https://kubernetes.github.io/ingress-nginx/kubectl-plugin/

https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md

https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.23.0

https://github.com/kubernetes/ingress-nginx/pull/3686

https://github.com/kubernetes/ingress-nginx/pull/2385

https://github.com/kubernetes/ingress-nginx/pull/3779