那些比你走得远的人，并不比你聪慧，只是每天多走了一点。坚持，是最强大的力量。

ETCD使用自签证书随记

发表于 更新于 分类于

ETCD使用自签证书汇总

证书类型介绍

  • client certificate 用于通过服务器验证客户端。例如etcdctl，etcd proxy，fleetctl或docker客户端。
  • server certificate 由服务器使用，并由客户端验证服务器身份。例如docker服务器或kube-apiserver。
  • peer certificate 由 etcd 集群成员使用，供它们彼此之间通信使用。

下载工具

1
2
3
curl -s -L -o /usr/bin/cfssl https://github.com/cloudflare/cfssl/releases/download/v1.6.0/cfssl_1.6.0_linux_amd64
curl -s -L -o /usr/bin/cfssljson https://github.com/cloudflare/cfssl/releases/download/v1.6.0/cfssljson_1.6.0_linux_amd64
chmod +x /usr/bin/{cfssl,cfssljson}

生成证书

配置CA选项：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
mkdir -p /data/ds/data/etcd-data/cfssl
etcd-data/cfssl

cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "43800h"
    },
    "profiles": {
      "server": {
        "expiry": "43800h",
        "usages": [
          "signing",
          "key encipherment",
          "server auth"
        ]
      },
      "client": {
        "expiry": "43800h",
        "usages": [
          "signing",
          "key encipherment",
          "client auth"
        ]
      },
      "peer": {
        "expiry": "43800h",
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ]
      }
    }
  }
}
EOF
cat > ca-csr.json <<EOF
{
  "CN": "Etcd CA",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "LN",
      "ST": "DA LIAN"
    }
  ]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

生成server证书：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
cat > server.json <<EOF
{
  "CN": "Etcd Server",
  "hosts": [
    "127.0.0.1",
    "etcd1.codeflag.cn",
    "etcd2.codeflag.cn",
    "etcd3.codeflag.cn"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "LN",
      "ST": "DA LIAN"
    }
  ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server.json | cfssljson -bare server

生成peer证书：需要修改hosts为主机的IP或域名

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
cat > member.json <<EOF
{
  "CN": "Etcd Server",
  "hosts": [
	"192.168.1.111",
    "etcd1.codeflag.cn"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "LN",
      "ST": "DA LIAN"
    }
  ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer member.json | cfssljson -bare member

生成client证书：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
cat > client.json <<EOF
{
  "CN": "Etcd Client",
  "hosts": [
    ""
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "LN",
      "ST": "DA LIAN"
    }
  ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client

将所有pem文件复制到/etc/etcd/etcd-ca/目录下：

1
2
mkdir -p /etc/etcd/etcd-ca/
cp *.pem /etc/etcd/etcd-ca/

配置Etcd变量

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
vim /etc/etcd/etcd.conf 
# [member]
ETCD_NAME=etcd1
ETCD_DATA_DIR="/var/lib/etcd/data"
ETCD_LISTEN_PEER_URLS="https://0.0.0.0:2380"
ETCD_LISTEN_CLIENT_URLS="https://0.0.0.0:2379"
#[cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://etcd1-codeflag.cn:2380"
ETCD_INITIAL_CLUSTER="etcd1=https://etcd1-codeflag.cn:2380,etcd2=https://etcd2-codeflag.cn:2380,etcd3=https://etcd3-codeflag.cn:2380,etcd4=https://etcd4-codeflag.cn:2380,etcd5=https://etcd5-codeflag.cn:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="https://etcd1-codeflag.cn:2379"
#[security]
ETCD_CERT_FILE="/etc/etcd/etcd-ca/server.pem"
ETCD_KEY_FILE="/etc/etcd/etcd-ca/server-key.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_TRUSTED_CA_FILE="/etc/etcd/etcd-ca/ca.pem"
ETCD_AUTO_TLS="true"
ETCD_PEER_CERT_FILE="/etc/etcd/etcd-ca/member.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/etcd-ca/member-key.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/etcd-ca/ca.pem"
ETCD_PEER_AUTO_TLS="true"

配置Etcd启动脚本

安装好后，系统会自动生成etcd.service文件（路径为/usr/lib/systemd/system/），修改对应的配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
User=etcd
# set GOMAXPROCS to number of processors
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd \
--name=\"${ETCD_NAME}\" \
--data-dir=\"${ETCD_DATA_DIR}\" \
--listen-peer-urls=\"${ETCD_LISTEN_PEER_URLS}\" \
--advertise-client-urls=\"${ETCD_ADVERTISE_CLIENT_URLS}\" \
--initial-cluster-token=\"${ETCD_INITIAL_CLUSTER_TOKEN}\" \
--initial-cluster=\"${ETCD_INITIAL_CLUSTER}\"  \
--initial-cluster-state=\"${ETCD_INITIAL_CLUSTER_STATE}\" \
--listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\""

Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

增加etcdctl环境变量

1
2
3
4
5
6
cat > /etc/profile.d/etcd.sh <<EOF
export ETCDCTL_CA_FILE=/etc/etcd/etcd-ca/ca.pem
export ETCDCTL_KEY_FILE=`/etc/etcd/etcd-ca/client-key.pem
export ETCDCTL_CERT_FILE=`/etc/etcd/etcd-ca/client.pem/EOF
EOF
. /etc/profile.d/etcd.sh

如果使用域名访问, 添加hosts, 集群中每台主机均要添加:

1
2
3
192.168.1.111 etcd1.codeflag.cn
192.168.1.112 etcd2.codeflag.cn
192.168.1.113 etcd3.codeflag.cn

命令行启动etcd

--name etcd-server-01 \
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
//单机
etcd --name etcd-server-01 \
  --data-dir /var/lib/etcd \
  --listen-client-urls http://0.0.0.0:2379 \
  --advertise-client-urls http://0.0.0.0:2379 \

//集群1
etcd --name etcd1 \
  --data-dir /data/ds/data/etcd-data/ \
  --listen-client-urls https://192.168.1.111:2379,https://127.0.0.1:2379 \
  --advertise-client-urls https://192.168.1.111:2379 \
  --listen-peer-urls https://192.168.1.111:2380 \
  --initial-advertise-peer-urls https://192.168.1.111:2380 \
  --initial-cluster-state new \
  --initial-cluster etcd1=https://192.168.1.111:2380,etcd2=https://192.168.1.112:2380,etcd3=https://192.168.1.113:2380 \
  --initial-cluster-token etcd-cluster \
  --cert-file /data/ds/data/etcd-data/cfssl/server.pem \
  --key-file /data/ds/data/etcd-data/cfssl/server-key.pem \
  --client-cert-auth \
  --trusted-ca-file /data/ds/data/etcd-data/cfssl/ca.pem \
  --auto-tls \
  --peer-cert-file /data/ds/data/etcd-data/cfssl/member.pem \
  --peer-key-file /data/ds/data/etcd-data/cfssl/member-key.pem \
  --peer-client-cert-auth \
  --peer-trusted-ca-file /data/ds/data/etcd-data/cfssl/ca.pem \
  --peer-auto-tls \
  --auto-compaction-mode periodic \
  --auto-compaction-retention 24 \
  --max-request-bytes 33554432 \
  --quota-backend-bytes 6442450944 \
  --heartbeat-interval 250 \
  --election-timeout 2000 \
  --log-level info \
  --logger zap \
  --log-outputs stderr 


//集群2
etcd --name etcd2 \
  --data-dir /data/ds/data/etcd-data/ \
  --listen-client-urls https://192.168.1.112:2379,https://127.0.0.1:2379 \
  --advertise-client-urls https://192.168.1.112:2379 \
  --listen-peer-urls https://192.168.1.112:2380 \
  --initial-advertise-peer-urls https://192.168.1.112:2380 \
  --initial-cluster-state new \
  --initial-cluster etcd1=https://192.168.1.111:2380,etcd2=https://192.168.1.112:2380,etcd3=https://192.168.1.113:2380 \
  --initial-cluster-token etcd-cluster \
  --cert-file /data/ds/data/etcd-data/cfssl/server.pem \
  --key-file /data/ds/data/etcd-data/cfssl/server-key.pem \
  --client-cert-auth \
  --trusted-ca-file /data/ds/data/etcd-data/cfssl/ca.pem \
  --auto-tls \
  --peer-cert-file /data/ds/data/etcd-data/cfssl/member.pem \
  --peer-key-file /data/ds/data/etcd-data/cfssl/member-key.pem \
  --peer-client-cert-auth \
  --peer-trusted-ca-file /data/ds/data/etcd-data/cfssl/ca.pem \
  --peer-auto-tls \
  --auto-compaction-mode periodic \
  --auto-compaction-retention 24 \
  --max-request-bytes 33554432 \
  --quota-backend-bytes 6442450944 \
  --heartbeat-interval 250 \
  --election-timeout 2000 \
  --log-level info \
  --logger zap \
  --log-outputs stderr 

//集群3
etcd --name etcd3 \
  --data-dir /data/ds/data/etcd-data/ \
  --listen-client-urls https://192.168.1.113:2379,https://127.0.0.1:2379 \
  --advertise-client-urls https://192.168.1.113:2379 \
  --listen-peer-urls https://192.168.1.113:2380 \
  --initial-advertise-peer-urls https://192.168.1.113:2380 \
  --initial-cluster-state new \
  --initial-cluster etcd1=https://192.168.1.111:2380,etcd2=https://192.168.1.112:2380,etcd3=https://192.168.1.113:2380 \
  --initial-cluster-token etcd-cluster \
  --cert-file /data/ds/data/etcd-data/cfssl/server.pem \
  --key-file /data/ds/data/etcd-data/cfssl/server-key.pem \
  --client-cert-auth \
  --trusted-ca-file /data/ds/data/etcd-data/cfssl/ca.pem \
  --auto-tls \
  --peer-cert-file /data/ds/data/etcd-data/cfssl/member.pem \
  --peer-key-file /data/ds/data/etcd-data/cfssl/member-key.pem \
  --peer-client-cert-auth \
  --peer-trusted-ca-file /data/ds/data/etcd-data/cfssl/ca.pem \
  --peer-auto-tls \
  --auto-compaction-mode periodic \
  --auto-compaction-retention 24 \
  --max-request-bytes 33554432 \
  --quota-backend-bytes 6442450944 \
  --heartbeat-interval 250 \
  --election-timeout 2000 \
  --log-level info \
  --logger zap \
  --log-outputs stderr

一些参数的说明

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
--listen-client-urls
用于监听客户端消息,必须设置为真实ip地址，如果机器为云主机，可以设置为云主机的私有ip地址或0.0.0.0(代表监听所有地址),不能设	置为公网ip地址，因为云主机的公网ip是虚拟的，如果设置为公网ip会报“bind: cannot assign requested address”的错
--listen-peer-urls
用于监听其他member发送过来的消息，跟listen-client-urls一样，必须设置为真实ip地址,如果机器为云主机,不能设置为公网ip
--initial-advertise-peer-urls
用于监听其他member同步信号，该地址其他member必须能直接访问，所以如果是云主机该地址必须设置为云主机的公网ip地址
--initial-cluster
群集列表，该列表中的值必须跟各个member的initial-advertise-peer-urls值一样
--advertise-client-urls
建议使用的客户端通信url,该值用于etcd代理或etcd成员与etcd节点通信。
--initial-cluster-token
节点的token值，设置该值后集群将生成唯一id,并为每个节点也生成唯一id,当使用相同配置文件再启动一个集群时，只要该token值不一样，etcd集群就不会相互影响。
--initial-cluster
也就是集群中所有的initial-advertise-peer-urls 的合集
--initial-cluster-state new
新建集群的标志，初始化状态使用 new，建立之后改此值为 existing

查看集群状态

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 显示集群主机状态
etcdctl \
  --write-out=table \
  --cacert=/etc/etcd/etcd-ca/ca.pem \
  --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
  --cert=/etc/etcd/etcd-ca/server.pem \
  --key=/etc/etcd/etcd-ca/server-key.pem \
  endpoint status     

# 显示集群主机列表
etcdctl \
  --write-out=table \
  --cacert=/etc/etcd/etcd-ca/ca.pem \
  --cert=/etc/etcd/etcd-ca/server.pem \
  --key=/etc/etcd/etcd-ca/server-key.pem \
  member list 

# 删除集群中的主机
etcdctl \                     
  --cacert=/data/ds/data/etcd-data/cfssl/ca.pem \
  --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
  --cert=/data/ds/data/etcd-data/cfssl/server.pem \
  --key=/data/ds/data/etcd-data/cfssl/server-key.pem \
  member remove 5838b30ed275b54

docker 命令部署

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
docker rm -f etcd && docker run \
    -p 2379:2379 \
    -p 2380:2380 \
    --mount type=bind,source=/tmp/etcd-data,destination=/etcd-data \
    --name etcd \
    gcr.io/etcd-development/etcd:v3.5.0 \
    /usr/local/bin/etcd \
        --name etcd1 \
        --data-dir /etcd-data \
      	--listen-client-urls https://192.168.1.111:2379
      	--advertise-client-urls https://192.168.1.111:2379
      	--listen-peer-urls https://192.168.1.111:2380"
      	--initial-advertise-peer-urls https://192.168.1.111:2380"
      	--initial-cluster etcd1=https://192.168.1.111:2380,etcd2=https://192.168.1.112:2380,etcd3=https://192.168.1.113:2380
        --initial-cluster-token token \
        --initial-cluster-state new \
        --cert-file /etcd-data/cfssl/server.pem \
        --key-file /etcd-data/cfssl/server-key.pem \
        --peer-cert-file /etcd-data/cfssl/member.pem \
        --peer-key-file /etcd-data/cfssl/member-key.pem \
        --peer-client-cert-auth \
        --client-cert-auth \
        --trusted-ca-file /etcd-data/cfssl/ca.pem \
        --auto-tls \
        --auto-compaction-mode periodic \
        --auto-compaction-retention 24 \
        --max-request-bytes 33554432 \
        --quota-backend-bytes 6442450944 \
        --heartbeat-interval 250 \
        --election-timeout 2000 \
        --log-level info \
        --logger zap \
        --log-outputs stderr

docker-compose 部署

集群内每台主机要修改 –name –urls

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
etcd:
  container_name: etcd
  image: ccr.ccs.tencentyun.com/kuaixue/etcd:3.5.0
  network_mode: host
  volumes:
    - ./data/etcd-data:/etcd-data:rw
  restart: always
  environment:
    ETCDCTL_API: 3
    TZ: Asia/Shanghai
  entrypoint: /usr/local/bin/etcd
  command:
    - "--name=etcd1"
    - "--data-dir=/etcd-data"
    - "--listen-client-urls=https://192.168.1.111:2379"
    - "--advertise-client-urls=https://192.168.1.111:2379"
    - "--listen-peer-urls=https://192.168.1.111:2380"
    - "--initial-advertise-peer-urls=https://192.168.1.111:2380"
    - "--initial-cluster=etcd1=https://192.168.1.111:2380,etcd2=https://192.168.1.112:2380,etcd3=https://192.168.1.113:2380"
    - "--initial-cluster-token=token"
    - "--initial-cluster-state=new"
    - "--cert-file=/etcd-data/cfssl/server.pem"
    - "--key-file=/etcd-data/cfssl/server-key.pem"
    - "--peer-trusted-ca-file=/etcd-data/cfssl/ca.pem"
    - "--peer-client-cert-auth"
    - "--peer-cert-file=/etcd-data/cfssl/member.pem"
    - "--peer-key-file=/etcd-data/cfssl/member-key.pem"
    - "--client-cert-auth"
    - "--trusted-ca-file=/etcd-data/cfssl/ca.pem"
    - "--auto-tls"
    - "--auto-compaction-mode=periodic"
    - "--auto-compaction-retention=24"
    - "--max-request-bytes=33554432"
    - "--quota-backend-bytes=6442450944"
    - "--heartbeat-interval=250"
    - "--election-timeout=2000"
    - "--log-level=info"
    - "--logger=zap"
    - "--log-outputs=stderr"

使用.env配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
# .env
################################### Etcd ################################
NET_OUTSIDE=202.202.202.202
NET_INSIDE=192.168.1.111

ETCD_NAME=etcd1
ETCD_DATA_DIR=./data/etcd-data
ETCD_LISTEN_CLIENT_URLS=https://192.168.1.111:2379,http://127.0.0.1:2379
ETCD_ADVERTISE_CLIENT_URLS=https://192.168.1.111:2379

ETCD_LISTEN_PEER_URLS=https://192.168.1.111:2380
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.1.111:2380
ETCD_INITIAL_CLUSTER=etcd1=https://192.168.1.111:2380,etcd2=https://192.168.1.112:2380,etcd3=https://192.168.1.113:2380
ETCD_INITIAL_CLUSTER_TOKEN=token
ETCD_INITIAL_CLUSTER_STATE=existing

ETCD_CERT_FILE=/etcd-data/cfssl/server.pem
ETCD_KEY_FILE=/etcd-data/cfssl/server-key.pem
ETCD_TRUSTED_CA_FILE=/etcd-data/cfssl/ca.pem
ETCD_PEER_CERT_FILE=/etcd-data/cfssl/server.pem
ETCD_PEER_KEY_FILE=/etcd-data/cfssl/server-key.pem
ETCD_PEER_TRUSTED_CA_FILE=/etcd-data/cfssl/ca.pem
ETCD_PEER_CLIENT_CERT_AUTH=true
ETCD_CLIENT_CERT_AUTH=true
ETCD_AUTO_TLS=true
ETCD_AUTO_COMPACTION_MODE=periodic
ETCD_AUTO_COMPACTION_RETENTION=24
ETCD_MAX_REQUEST_BYTES=33554432
ETCD_QUOTA_BACKEND_BYTES=6442450944
ETCD_HEARTBEAT_INTERVAL=250
ETCD_ELECTION_TIMEOUT=2000
ETCD_LOG_LEVEL=info
ETCD_LOGGER=zap
ETCD_LOG_OUTPUTS=stderr

# docker-compose.yml 
  etcd:
    container_name: etcd
    image: ccr.ccs.tencentyun.com/kuaixue/etcd:3.5.0
    network_mode: host
    volumes:
      - ${ETCD_DATA_DIR}:/etcd-data:rw
    restart: always
    environment:
      ETCDCTL_API: 3
      TZ: Asia/Shanghai
    entrypoint: /usr/local/bin/etcd
    command:
      - --name=${ETCD_NAME}
      - --data-dir=/etcd-data
      - --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS}
      - --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS}
      - --listen-peer-urls=${ETCD_LISTEN_PEER_URLS}
      - --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS}
      - --initial-cluster=${ETCD_INITIAL_CLUSTER}
      - --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN}
      - --initial-cluster-state=${ETCD_INITIAL_CLUSTER_STATE}
      - --cert-file=${ETCD_CERT_FILE}
      - --key-file=${ETCD_KEY_FILE}
      - --trusted-ca-file=${ETCD_TRUSTED_CA_FILE}
      - --peer-trusted-ca-file=${ETCD_PEER_TRUSTED_CA_FILE}
      - --peer-cert-file=${ETCD_PEER_CERT_FILE}
      - --peer-key-file=${ETCD_PEER_KEY_FILE}
      - --peer-client-cert-auth
      - --client-cert-auth
      - --auto-tls
      - --auto-compaction-mode=${ETCD_AUTO_COMPACTION_MODE}
      - --auto-compaction-retention=${ETCD_AUTO_COMPACTION_RETENTION}
      - --max-request-bytes=${ETCD_MAX_REQUEST_BYTES}
      - --quota-backend-bytes=${ETCD_QUOTA_BACKEND_BYTES}
      - --heartbeat-interval=${ETCD_HEARTBEAT_INTERVAL}
      - --election-timeout=${ETCD_ELECTION_TIMEOUT}
      - --log-level=${ETCD_LOG_LEVEL}
      - --logger=${ETCD_LOGGER}
      - --log-outputs=${ETCD_LOG_OUTPUTS}

从网络加载配置

1
--discovery=https://discovery.etcd.io/2b3b04efc64b19323ba8f60dd62680a0

从集群中删除一台主机

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
etcdctl \
  --write-out=table \
  --cacert=/data/ds/data/etcd-data/cfssl/ca.pem \
  --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
  --cert=/data/ds/data/etcd-data/cfssl/server.pem \
  --key=/data/ds/data/etcd-data/cfssl/server-key.pem \
  member list    
+------------------+---------+-------+----------------------------+----------------------------+------------+
|        ID        | STATUS  | NAME  |         PEER ADDRS         |        CLIENT ADDRS        | IS LEARNER |
+------------------+---------+-------+----------------------------+----------------------------+------------+
|  5838b30ed275b54 | started | etcd1 | https://192.168.1.111:2380 | https://192.168.1.111:2379 |      false |
| 65307ea188059cec | started | etcd2 | https://192.168.1.112:2380 | https://192.168.1.112:2379 |      false |
| 742cd8ca4086ab80 | started | etcd3 | https://192.168.1.113:2380 | https://192.168.1.113:2379 |      false |
+------------------+---------+-------+----------------------------+----------------------------+------------+

etcdctl \
  --cacert=/data/ds/data/etcd-data/cfssl/ca.pem \
  --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
  --cert=/data/ds/data/etcd-data/cfssl/server.pem \
  --key=/data/ds/data/etcd-data/cfssl/server-key.pem \
  member remove 5838b30ed275b54

etcdctl \                                                                       
  --write-out=table \
  --cacert=/data/ds/data/etcd-data/cfssl/ca.pem \
  --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
  --cert=/data/ds/data/etcd-data/cfssl/server.pem \
  --key=/data/ds/data/etcd-data/cfssl/server-key.pem \
  endpoint status
{"level":"warn","ts":"2021-07-30T07:40:37.663+0800","logger":"etcd-client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0003bea80/#initially=[https://192.168.1.111:2379;https://192.168.1.112:2379;https://192.168.1.113:2379]","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing dial tcp 192.168.1.111:2379: connect: connection refused\""}
Failed to get the status of endpoint https://192.168.1.111:2379 (context deadline exceeded)
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.1.112:2379 | 65307ea188059cec |   3.5.0 |   20 kB |      true |      false |         3 |         10 |                 10 |        |
| https://192.168.1.113:2379 | 742cd8ca4086ab80 |   3.5.0 |   20 kB |     false |      false |         3 |         10 |                 10 |        |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

将删除的主机再添加回集群

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
etcdctl \
  --cacert=/data/ds/data/etcd-data/cfssl/ca.pem \
  --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
  --cert=/data/ds/data/etcd-data/cfssl/server.pem \
  --key=/data/ds/data/etcd-data/cfssl/server-key.pem \
  member add etcd1 --peer-urls=https://192.168.1.111:2380 
Member 4c23d6636feaae36 added to cluster a30ae613ea53be49

ETCD_NAME="etcd1"
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.111:2380,etcd2=https://192.168.1.112:2380,etcd3=https://192.168.1.113:2380"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.111:2380"
ETCD_INITIAL_CLUSTER_STATE="existing"

etcdctl \
  --write-out=table \
  --cacert=/data/ds/data/etcd-data/cfssl/ca.pem \
  --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
  --cert=/data/ds/data/etcd-data/cfssl/server.pem \
  --key=/data/ds/data/etcd-data/cfssl/server-key.pem \
  member list
+------------------+-----------+-------+----------------------------+----------------------------+------------+
|        ID        |  STATUS   | NAME  |         PEER ADDRS         |        CLIENT ADDRS        | IS LEARNER |
+------------------+-----------+-------+----------------------------+----------------------------+------------+
| 4c23d6636feaae36 | unstarted |       | https://192.168.1.111:2380 |                            |      false |
| 65307ea188059cec |   started | etcd2 | https://192.168.1.112:2380 | https://192.168.1.112:2379 |      false |
| 742cd8ca4086ab80 |   started | etcd3 | https://192.168.1.113:2380 | https://192.168.1.113:2379 |      false |
+------------------+-----------+-------+----------------------------+----------------------------+------------+

docker rm -f etcd && cd /data/ds/ && rm -rf /data/ds/data/etcd-data/member && docker-compose up etcd

etcdctl \
  --write-out=table \
  --cacert=/data/ds/data/etcd-data/cfssl/ca.pem \
  --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
  --cert=/data/ds/data/etcd-data/cfssl/server.pem \
  --key=/data/ds/data/etcd-data/cfssl/server-key.pem \
  member list
+------------------+---------+-------+----------------------------+----------------------------+------------+
|        ID        | STATUS  | NAME  |         PEER ADDRS         |        CLIENT ADDRS        | IS LEARNER |
+------------------+---------+-------+----------------------------+----------------------------+------------+
| 4c23d6636feaae36 | started | etcd1 | https://192.168.1.111:2380 | https://192.168.1.111:2379 |      false |
| 65307ea188059cec | started | etcd2 | https://192.168.1.112:2380 | https://192.168.1.112:2379 |      false |
| 742cd8ca4086ab80 | started | etcd3 | https://192.168.1.113:2380 | https://192.168.1.113:2379 |      false |
+------------------+---------+-------+----------------------------+----------------------------+------------+

etcdctl \
  --write-out=table \
  --cacert=/data/ds/data/etcd-data/cfssl/ca.pem \
  --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
  --cert=/data/ds/data/etcd-data/cfssl/server.pem \
  --key=/data/ds/data/etcd-data/cfssl/server-key.pem \
  endpoint status                                         
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|          ENDPOINT          |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.1.111:2379 | 53cf44940878a47c |   3.5.0 |   20 kB |     false |      false |         2 |         21 |                 21 |        |
| https://192.168.1.112:2379 | 65307ea188059cec |   3.5.0 |   20 kB |      true |      false |         2 |         21 |                 21 |        |
| https://192.168.1.113:2379 | 742cd8ca4086ab80 |   3.5.0 |   20 kB |     false |      false |         2 |         21 |                 21 |        |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

RESTFUL API V3

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.113:2379/v3/kv/txn \
  -X POST \
  -d '{"compare":[{"target":"CREATE","key":"Zm9v","createRevision":"2"}],"success":[{"requestPut":{"key":"Zm9v","value":"YmFy"}}]}'


curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.113:2379/v3/kv/watch \
  -X POST -d '{"create_request": {"key":"Zm9v"} }' &

curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.113:2379/v3/kv/put \
  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'


curl --cacert ./ca.pem --cert ./client.pem --key ./client-key.pem https://172.12.0.16:2379/version
curl --cacert ./ca.pem --cert ./client.pem --key ./client-key.pem https://192.168.1.112:2379/version

# Put and get keys
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.111:2379/v3/kv/range \
  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
# {"header":{"cluster_id":"11748455551159483977","member_id":"397314234392206164","revision":"51","raft_term":"2"},"kvs":[{"key":"Zm9v","create_revision":"48","mod_revision":"51","version":"4","value":"YmFy"}],"count":"1"}#                             

curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/kv/put \
  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
# {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"3"}}

curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/kv/range \
  -X POST -d '{"key": "Zm9v"}'
# {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"3"},"kvs":[{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}],"count":"1"}

# get all keys prefixed with "foo"
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/kv/range \
  -X POST -d '{"key": "Zm9v", "range_end": "Zm9w"}'
# {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"3"},"kvs":[{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}],"count":"1"}

# get the auth token for the root user
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/auth/authenticate \
  -X POST -d '{"name": "root", "password": "pass"}'
# {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"},"token":"sssvIpwfnLAcWAQH.9"}

curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/kv/put \
  -H 'Authorization : sssvIpwfnLAcWAQH.9' \
  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
# {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"2","raft_term":"2"}}

# Watch keys
curl --cacert ca.pem --cert client.pem --key client-key.pem -N https://192.168.1.112:2379/v3/watch \
  -X POST -d '{"create_request": {"key":"Zm9v"} }' &
# {"result":{"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"1","raft_term":"2"},"created":true}}

curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/kv/put \
  -X POST -d '{"key": "Zm9v", "value": "YmFy"}' 
# {"result":{"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"2"},"events":[{"kv":{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}}]}}

# Transactions
# target CREATE
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/kv/txn \
  -X POST \
  -d '{"compare":[{"target":"CREATE","key":"Zm9v","createRevision":"2"}],"success":[{"requestPut":{"key":"Zm9v","value":"YmFy"}}]}'
# {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"3","raft_term":"2"},"succeeded":true,"responses":[{"response_put":{"header":{"revision":"3"}}}]}

# target VERSION
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/kv/txn \
  -X POST \
  -d '{"compare":[{"version":"4","result":"EQUAL","target":"VERSION","key":"Zm9v"}],"success":[{"requestRange":{"key":"Zm9v"}}]}'
# {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"6","raft_term":"3"},"succeeded":true,"responses":[{"response_range":{"header":{"revision":"6"},"kvs":[{"key":"Zm9v","create_revision":"2","mod_revision":"6","version":"4","value":"YmF6"}],"count":"1"}}]}

# Authentication
# create root user
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/auth/user/add \
  -X POST -d '{"name": "root", "password": "pass"}'
# {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"}}

# create root role
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/auth/role/add \
  -X POST -d '{"name": "root"}'
# {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"}}

# grant root role
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/auth/user/grant \
  -X POST -d '{"user": "root", "role": "root"}'
# {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"}}

# enable auth
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/auth/enable -X POST -d '{}'
# {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"}}

# Authenticate with etcd for an authentication token using /v3/auth/authenticate:
# get the auth token for the root user
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/auth/authenticate \
  -X POST -d '{"name": "root", "password": "pass"}'
# {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"},"token":"sssvIpwfnLAcWAQH.9"}
curl --cacert ca.pem --cert client.pem --key client-key.pem -L https://192.168.1.112:2379/v3/kv/put \
  -H 'Authorization : sssvIpwfnLAcWAQH.9' \
  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
# {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"2","raft_term":"2"}}

参考文档