secrets 管理工具 Vault 的介绍、安装及使用

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

version: "3.3"
services:
  vault:
    # 文档：https://hub.docker.com/_/vault
    image: vault:1.6.0
    container_name: vault
    ports:
      # rootless 容器，内部不能使用标准端口 443
      - "443:8200"
    restart: always
    volumes:
      # 审计日志存储目录（`file` audit backend）
      - ./logs:/vault/logs
      # 当使用 file data storage 插件时，数据被存储在这里。默认不往这写任何数据。
      - ./file:/vault/file
      # vault 配置
      - ./config.hcl:/vault/config/config.hcl
      # TLS 证书
      - ./certs:/certs
    # vault 需要锁定内存以防止敏感值信息被交换(swapped)到磁盘中
    # 为此需要添加如下 capability
    cap_add:
      - IPC_LOCK
    # 必须设定 entrypoint，否则 vault 容器默认以 development 模式运行
    entrypoint: vault server -config /vault/config/config.hcl

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

ui = true

// 使用文件做数据存储（单节点）
storage "file" {
  path    = "/vault/file"
}

listener "tcp" {
  address = "[::]:8200"

  tls_disable = false
  tls_cert_file = "/certs/server.crt"
  tls_key_file  = "/certs/server.key"
}

1
2
3
4
5
6

# 添加 valut 仓库
helm repo add hashicorp https://helm.releases.hashicorp.com
# 查看 vault 版本号
helm search repo hashicorp/vault -l | head
# 下载某个版本号的 vault
helm pull hashicorp/vault --version  0.11.0 --untar

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

global:
  # enabled is the master enabled switch. Setting this to true or false
  # will enable or disable all the components within this chart by default.
  enabled: true
  # TLS for end-to-end encrypted transport
  tlsDisable: false

injector:
  # True if you want to enable vault agent injection.
  enabled: true

  replicas: 1

  # If true, will enable a node exporter metrics endpoint at /metrics.
  metrics:
    enabled: false

  # Mount Path of the Vault Kubernetes Auth Method.
  authPath: "auth/kubernetes"

  certs:
    # secretName is the name of the secret that has the TLS certificate and
    # private key to serve the injector webhook. If this is null, then the
    # injector will default to its automatic management mode that will assign
    # a service account to the injector to generate its own certificates.
    secretName: null

    # caBundle is a base64-encoded PEM-encoded certificate bundle for the
    # CA that signed the TLS certificate that the webhook serves. This must
    # be set if secretName is non-null.
    caBundle: ""

    # certName and keyName are the names of the files within the secret for
    # the TLS cert and private key, respectively. These have reasonable
    # defaults but can be customized if necessary.
    certName: tls.crt
    keyName: tls.key

server:
  # Resource requests, limits, etc. for the server cluster placement. This
  # should map directly to the value of the resources field for a PodSpec.
  # By default no direct resource request is made.

  # Enables a headless service to be used by the Vault Statefulset
  service:
    enabled: true
    # Port on which Vault server is listening
    port: 8200
    # Target port to which the service should be mapped to
    targetPort: 8200

  # This configures the Vault Statefulset to create a PVC for audit
  # logs.  Once Vault is deployed, initialized and unseal, Vault must
  # be configured to use this for audit logs.  This will be mounted to
  # /vault/audit
  # See https://www.vaultproject.io/docs/audit/index.html to know more
  auditStorage:
    enabled: false

  # Run Vault in "HA" mode. There are no storage requirements unless audit log
  # persistence is required.  In HA mode Vault will configure itself to use Consul
  # for its storage backend.  The default configuration provided will work the Consul
  # Helm project by default.  It is possible to manually configure Vault to use a
  # different HA backend.
  ha:
    enabled: true
    replicas: 3

    # Set the api_addr configuration for Vault HA
    # See https://www.vaultproject.io/docs/configuration#api_addr
    # If set to null, this will be set to the Pod IP Address
    apiAddr: null

    # config is a raw string of default configuration when using a Stateful
    # deployment. Default is to use a Consul for its HA storage backend.
    # This should be HCL.

    # Note: Configuration files are stored in ConfigMaps so sensitive data
    # such as passwords should be either mounted through extraSecretEnvironmentVars
    # or through a Kube secret.  For more information see:
    # https://www.vaultproject.io/docs/platform/k8s/helm/run#protecting-sensitive-vault-configurations
    config: |
      ui = true

      listener "tcp" {
        address = "[::]:8200"
        cluster_address = "[::]:8201"

        # 注意，这个值要和 helm 的参数 global.tlsDisable 一致
        tls_disable = false
        tls_cert_file = "/etc/certs/vault.crt"
        tls_key_file  = "/etc/certs/vault.key"
      }

      # storage "postgresql" {
      #   connection_url = "postgres://username:password@<host>:5432/vault?sslmode=disable"
      #   ha_enabled = true
      # }

      service_registration "kubernetes" {}

      # Example configuration for using auto-unseal, using AWS KMS. 
      # the cluster must have a service account that is authorized to access AWS KMS, through an IAM Role.
      # seal "awskms" {
      #   region     = "us-east-1"
      #   kms_key_id = "<some-key-id>"
      #   默认情况下插件会使用 awskms 的公网 endpoint，但是也可以使用如下参数，改用自行创建的 vpc 内网 endpoint
      #   endpoint   = "https://<vpc-endpoint-id>.kms.us-east-1.vpce.amazonaws.com"
      # }      

  # Definition of the serviceAccount used to run Vault.
  # These options are also used when using an external Vault server to validate
  # Kubernetes tokens.
  serviceAccount:
    create: true
    name: "vault"
    annotations:
      # 如果要使用 auto unseal 的话，这个填写拥有 awskms 权限的 AWS IAM Role
      eks.amazonaws.com/role-arn: <role-arn>

# Vault UI
ui:
  enabled: true
  publishNotReadyAddresses: true
  serviceType: ClusterIP
  activeVaultPodOnly: true
  externalPort: 8200

1
2
3

kubectl create namespace vault
# 安装/升级 valut
helm upgrade --install vault ./vault --namespace vault -f custom-values.yaml

1
2
3
4
5

# 每个实例都需要解封三次！
## Unseal the first vault server until it reaches the key threshold
$ kubectl exec -ti vault-0 -- vault operator unseal # ... Unseal Key 1
$ kubectl exec -ti vault-0 -- vault operator unseal # ... Unseal Key 2
$ kubectl exec -ti vault-0 -- vault operator unseal # ... Unseal Key 3

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "VaultKMSUnseal",
      "Effect": "Allow",
      "Action": ["kms:Decrypt", "kms:Encrypt", "kms:DescribeKey"],
      "Resource": "*"
    }
  ]
}

1
2
3
4

# 初始化命令和普通模式并无不同
kubectl exec -ti vault-0 -- vault operator init
# 会打印出一个 root token，以及五个 Recovery Key（而不是 Unseal Key）
# Recover Key 不再用于解封，但是重新生成 root token 等操作仍然会需要用到它.

1
2
3
4
5
6
7
8

path "local/*" {
  capabilities = ["read", "list"]
}

// 允许创建 child token
path "auth/token/create" {
  capabilities = ["create", "read", "update", "delete", "list"]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# To list policies - Step 3
path "sys/policy"
{
  capabilities = ["read"]
}

# Create and manage ACL policies broadly across Vault
path "sys/policy/*"
{
  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
}

# List, create, update, and delete key/value secrets
path "secret/*"
{
  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
}

path "auth/kubernetes/role/*"
{
  capabilities = ["create", "read", "update", "list"]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# 配置身份认证需要在 vault pod 中执行，启动 vault-0 的交互式会话
kubectl exec -n vault -it vault-0 -- /bin/sh
export VAULT_TOKEN='<your-root-token>'
export VAULT_ADDR='http://localhost:8200'

# 启用 Kubernetes 身份验证
vault auth enable kubernetes

# kube-apiserver API 配置，vault 需要通过 kube-apiserver 完成对 serviceAccount 的身份验证
vault write auth/kubernetes/config \
    token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
    kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" \
    kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

global:
  # enabled is the master enabled switch. Setting this to true or false
  # will enable or disable all the components within this chart by default.
  enabled: true
  # TLS for end-to-end encrypted transport
  tlsDisable: false

injector:
  # True if you want to enable vault agent injection.
  enabled: true

  replicas: 1

  # If multiple replicas are specified, by default a leader-elector side-car
  # will be created so that only one injector attempts to create TLS certificates.
  leaderElector:
    enabled: true
    image:
      repository: "gcr.io/google_containers/leader-elector"
      tag: "0.4"
    ttl: 60s

  # If true, will enable a node exporter metrics endpoint at /metrics.
  metrics:
    enabled: false

  # External vault server address for the injector to use. Setting this will
  # disable deployment of a  vault server along with the injector.
  # TODO 这里的 https ca.crt 要怎么设置？mTLS 又该如何配置？
  externalVaultAddr: "https://<external-vault-url>"

  # Mount Path of the Vault Kubernetes Auth Method.
  authPath: "auth/kubernetes"

  certs:
    # secretName is the name of the secret that has the TLS certificate and
    # private key to serve the injector webhook. If this is null, then the
    # injector will default to its automatic management mode that will assign
    # a service account to the injector to generate its own certificates.
    secretName: null

    # caBundle is a base64-encoded PEM-encoded certificate bundle for the
    # CA that signed the TLS certificate that the webhook serves. This must
    # be set if secretName is non-null.
    caBundle: ""

    # certName and keyName are the names of the files within the secret for
    # the TLS cert and private key, respectively. These have reasonable
    # defaults but can be customized if necessary.
    certName: tls.crt
    keyName: tls.key

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: vault-auth
  namespace: vault
---
apiVersion: v1
kind: Secret
metadata:
  name: vault-auth
  namespace: vault
  annotations:
    kubernetes.io/service-account.name: vault-auth
type: kubernetes.io/service-account-token
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: role-tokenreview-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
  - kind: ServiceAccount
    name: vault-auth
    namespace: vault

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

export VAULT_TOKEN='<your-root-token>'
export VAULT_ADDR='http://localhost:8200'

# 启用 Kubernetes 身份验证
vault auth enable kubernetes

# kube-apiserver API 配置，vault 需要通过 kube-apiserver 完成对 serviceAccount 的身份验证
# TOKEN_REVIEW_JWT: 就是我们前面创建的 secret `vault-auth`
TOKEN_REVIEW_JWT=$(kubectl -n vault get secret vault-auth -o go-template='{{ .data.token }}' | base64 --decode)
# kube-apiserver 的 ca 证书
KUBE_CA_CERT=$(kubectl -n vault config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 --decode)
# kube-apiserver 的 url
KUBE_HOST=$(kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.server}')

vault write auth/kubernetes/config \
        token_reviewer_jwt="$TOKEN_REVIEW_JWT" \
        kubernetes_host="$KUBE_HOST" \
        kubernetes_ca_cert="$KUBE_CA_CERT"

1
2
3
4
5
6
7
8

# 允许读取数据
path "my-app/data/*" {
   capabilities = ["read", "list"]
}
// 允许列出 myapp 中的所有数据(kv v2)
path "myapp/metadata/*" {
    capabilities = ["read", "list"]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: my-app
  name: my-app
  namespace: default
spec:
  minReadySeconds: 3
  progressDeadlineSeconds: 60
  revisionHistoryLimit: 3
  selector:
    matchLabels:
      app: my-app
  strategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      annotations:
        vault.hashicorp.com/agent-init-first: "true" # 是否使用 initContainer 提前初始化配置文件
        vault.hashicorp.com/agent-inject: "true"
        vault.hashicorp.com/secret-volume-path: vault
        vault.hashicorp.com/role: "my-app-role" # vault kubernetes 插件的 role 名称
        vault.hashicorp.com/agent-inject-template-config.json: |
          # 渲染模板的语法在后面介绍          
        vault.hashicorp.com/agent-limits-cpu: 250m
        vault.hashicorp.com/agent-requests-cpu: 100m
        # 包含 vault 配置的 configmap，可以做更精细的控制
        # vault.hashicorp.com/agent-configmap: my-app-vault-config
      labels:
        app: my-app
    spec:
      containers:
        - image: registry.svc.local/xx/my-app:latest
          imagePullPolicy: IfNotPresent
          # 此处省略若干配置...
      serviceAccountName: my-app-account

1
2
3
4
5
6
7
8

{
    {{ range secrets "<engine-name>/metadata/<service-name>/" }}
        "{{ printf "%s" . }}":
        {{ with secret (printf "<engine-name>/<service-name>/%s" .) }}
        {{ .Data.data | toJSONPretty }},
        {{ end }}
    {{ end }}
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

{
  "secret-a": {
    "a": "b",
    "c": "d"
  },
  "secret-b": {
    "v": "g",
    "r": "c"
  }
}