k8s部署中间件-rabbitmq

共计 10046 个字符，预计需要花费 26 分钟才能阅读完成。

standalone方式部署单节点

创建pvc

[root@k8s-master standalone]# cat >rabbitmq-pvc.yaml<<-'EOF'
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: rabbitmq-pvc
spec:
  storageClassName: nfs-storage
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1024Mi
EOF

创建deploy

[root@k8s-master standalone]# cat >rabbitmq-deploy.yaml<<-'EOF'
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rabbitmq
  labels:
    app: rabbitmq
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rabbitmq
  template:
    metadata:
      labels:
        app: rabbitmq
    spec:
      hostname: my-rabbit
      containers:
      - name: rabbitmq
        image: rabbitmq:3.8-management
        securityContext:
          runAsUser: 2000
          runAsNonRoot: true
          runAsGroup: 2000
        ports:
        - name: rabbitmq-web
          containerPort: 15672
        - name: rabbitmq-app
          containerPort: 5672
        volumeMounts:
        - name: persisitent-storage-rabbitmq
          mountPath: /var/lib/rabbitmq
        env:
        - name: RABBITMQ_DEFAULT_VHOST
          value: my_vhost
        - name: RABBITMQ_DEFAULT_USER
          value: admin
        - name: RABBITMQ_DEFAULT_PASS
          value: yourpassword
      volumes:
      - name: persisitent-storage-rabbitmq
        persistentVolumeClaim:
          claimName: rabbitmq-pvc
EOF

创建svc

[root@k8s-master standalone]# cat >rabbitmq-svc.yaml<<-'EOF'
apiVersion: v1
kind: Service
metadata:
  name: rabbitmq-web
spec:
  ports:
  - port: 15672
    nodePort: 31900
  selector:
    app: rabbitmq
  type: NodePort
---
apiVersion: v1
kind: Service
metadata:
  name: rabbitmq-app
spec:
  ports:
  - port: 5672
  selector:
    app: rabbitmq
EOF

部署集群模式

创建configmap配置rabbitmq.conf

[root@k8s-master cluster]# cat >rabbitmq-configmap.yaml<<-'EOF'
kind: ConfigMap
apiVersion: v1
metadata:
  name: rabbitmq-cluster-config
  namespace: default
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
data:
    enabled_plugins: |
      [rabbitmq_management,rabbitmq_peer_discovery_k8s].
    rabbitmq.conf: |
      default_user = admin
      default_pass = 123!@#
      ## Cluster formation. See https://www.rabbitmq.com/cluster-formation.html to learn more.
      cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s
      cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
      ## Should RabbitMQ node name be computed from the pod's hostname or IP address?
      ## IP addresses are not stable, so using [stable] hostnames is recommended when possible.
      ## Set to "hostname" to use pod hostnames.
      ## When this value is changed, so should the variable used to set the RABBITMQ_NODENAME
      ## environment variable.
      cluster_formation.k8s.address_type = hostname
      ## How often should node cleanup checks run?
      cluster_formation.node_cleanup.interval = 30
      ## Set to false if automatic removal of unknown/absent nodes
      ## is desired. This can be dangerous, see
      ##  * https://www.rabbitmq.com/cluster-formation.html#node-health-checks-and-cleanup
      ##  * https://groups.google.com/forum/#!msg/rabbitmq-users/wuOfzEywHXo/k8z_HWIkBgAJ
      cluster_formation.node_cleanup.only_log_warning = true
      cluster_partition_handling = autoheal
      ## See https://www.rabbitmq.com/ha.html#master-migration-data-locality
      queue_master_locator=min-masters
      ## See https://www.rabbitmq.com/access-control.html#loopback-users
      loopback_users.guest = false
      cluster_formation.randomized_startup_delay_range.min = 0
      cluster_formation.randomized_startup_delay_range.max = 2
      # default is rabbitmq-cluster's namespace
      # hostname_suffix
      cluster_formation.k8s.hostname_suffix = .rabbitmq-cluster.default.svc.cluster.local
      # memory
      vm_memory_high_watermark.absolute = 1GB
      # disk
      disk_free_limit.absolute = 2GB
EOF

创建svc服务

[root@k8s-master cluster]# cat >rabbitmq-service.yaml<<-'EOF'
kind: Service
apiVersion: v1
metadata:
  labels:
    app: rabbitmq-cluster
  name: rabbitmq-cluster
  namespace: default
spec:
  clusterIP: None
  ports:
  - name: rmqport
    port: 5672
    targetPort: 5672
  selector:
    app: rabbitmq-cluster

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: rabbitmq-cluster
  name: rabbitmq-cluster-manage
  namespace: default
spec:
  ports:
  - name: http
    port: 15672
    protocol: TCP
    targetPort: 15672
  selector:
    app: rabbitmq-cluster
  type: NodePort
EOF

创建rbac，为rabbimq提供权限获取集群endpoint信息

[root@k8s-master cluster]# cat >rabbitmq-rbac.yaml<<-'EOF'
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rabbitmq-cluster
  namespace: default
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: rabbitmq-cluster
  namespace: default
rules:
- apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: rabbitmq-cluster
  namespace: default
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rabbitmq-cluster
subjects:
- kind: ServiceAccount
  name: rabbitmq-cluster
  namespace: default
EOF

使用statfulset部署集群模式

[root@k8s-master cluster]# cat >rabbitmq-cluster-sts.yaml<<-'EOF'
kind: StatefulSet
apiVersion: apps/v1
metadata:
  labels:
    app: rabbitmq-cluster
  name: rabbitmq-cluster
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: rabbitmq-cluster
  serviceName: rabbitmq-cluster
  template:
    metadata:
      labels:
        app: rabbitmq-cluster
    spec:
      containers:
      - args:
        - -c
        - cp -v /etc/rabbitmq/rabbitmq.conf ${RABBITMQ_CONFIG_FILE}; exec docker-entrypoint.sh
          rabbitmq-server
        command:
        - sh
        env:
        - name: TZ
          value: 'Asia/Shanghai'
        - name: RABBITMQ_ERLANG_COOKIE
          value: 'SWvCP0Hrqv43NG7GybHC95ntCJKoW8UyNFWnBEWG8TY='
        - name: K8S_SERVICE_NAME
          value: rabbitmq-cluster
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: RABBITMQ_USE_LONGNAME
          value: "true"
        - name: RABBITMQ_NODENAME
          value: rabbit@$(POD_NAME).$(K8S_SERVICE_NAME).$(POD_NAMESPACE).svc.cluster.local
        - name: RABBITMQ_CONFIG_FILE
          value: /var/lib/rabbitmq/rabbitmq.conf
        image: rabbitmq:3.8.3-management
        imagePullPolicy: IfNotPresent
        livenessProbe:
          exec:
            command:
            - rabbitmq-diagnostics
            - status
          # See https://www.rabbitmq.com/monitoring.html for monitoring frequency recommendations.
          initialDelaySeconds: 60
          periodSeconds: 60
          timeoutSeconds: 15
        name: rabbitmq
        ports:
        - containerPort: 15672
          name: http
          protocol: TCP
        - containerPort: 5672
          name: amqp
          protocol: TCP
        readinessProbe:
          exec:
            command:
            - rabbitmq-diagnostics
            - status
          initialDelaySeconds: 20
          periodSeconds: 60
          timeoutSeconds: 10
        volumeMounts:
        - mountPath: /etc/rabbitmq
          name: config-volume
          readOnly: false
        - mountPath: /var/lib/rabbitmq
          name: rabbitmq-storage
          readOnly: false
        - name: timezone
          mountPath: /etc/localtime
          readOnly: true
      serviceAccountName: rabbitmq-cluster
      terminationGracePeriodSeconds: 30
      volumes:
      - name: config-volume
        configMap:
          items:
          - key: rabbitmq.conf
            path: rabbitmq.conf
          - key: enabled_plugins
            path: enabled_plugins
          name: rabbitmq-cluster-config
      - name: timezone
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
  volumeClaimTemplates:
  - metadata:
      name: rabbitmq-storage
    spec:
      accessModes:
      - ReadWriteMany
      storageClassName: "nfs-storage"
      resources:
        requests:
          storage: 2Gi
EOF

验证集群状态

[root@k8s-master cluster]# kubectl get svc
NAME                      TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)           AGE
kubernetes                ClusterIP   10.96.0.1       <none>        443/TCP           59d
nginx                     ClusterIP   None            <none>        80/TCP            5d5h
rabbitmq-cluster          ClusterIP   None            <none>        5672/TCP          89m
rabbitmq-cluster-manage   NodePort    10.96.217.185   <none>        15672:31630/TCP   89m
redis-svc                 ClusterIP   None            <none>        6379/TCP          5d5h
[root@k8s-master cluster]# kubectl get -f rabbitmq-configmap.yaml
NAME                      DATA   AGE
rabbitmq-cluster-config   2      85m
[root@k8s-master cluster]# kubectl get -f rabbitmq-rbac.yaml
NAME                              SECRETS   AGE
serviceaccount/rabbitmq-cluster   1         89m

NAME                                              CREATED AT
role.rbac.authorization.k8s.io/rabbitmq-cluster   2022-09-17T07:06:36Z

NAME                                                     ROLE                    AGE
rolebinding.rbac.authorization.k8s.io/rabbitmq-cluster   Role/rabbitmq-cluster   89m
[root@k8s-master cluster]# kubectl get -f rabbitmq-service.yaml
NAME                      TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)           AGE
rabbitmq-cluster          ClusterIP   None            <none>        5672/TCP          90m
rabbitmq-cluster-manage   NodePort    10.96.217.185   <none>        15672:31630/TCP   90m
[root@k8s-master cluster]# kubectl get -f rabbitmq-cluster-sts.yaml
NAME               READY   AGE
rabbitmq-cluster   3/3     86m
[root@k8s-master cluster]# kubectl get pvc |grep rabbi
rabbitmq-storage-rabbitmq-cluster-0   Bound    pvc-4fa770c5-951d-47da-9a59-58a6ec87a605   2Gi        RWX            nfs-storage    86m
rabbitmq-storage-rabbitmq-cluster-1   Bound    pvc-f870030f-2c4b-46e0-9aed-68f18446c8a9   2Gi        RWX            nfs-storage    84m
rabbitmq-storage-rabbitmq-cluster-2   Bound    pvc-dbf8322e-0ea1-4856-8dbf-d1f341486af3   2Gi        RWX            nfs-storage    82m

查看集群节点信息

[root@k8s-master cluster]# kubectl exec -it  rabbitmq-cluster-0 -- rabbitmqctl cluster_status
Cluster status of node rabbit@rabbitmq-cluster-0.rabbitmq-cluster.default.svc.cluster.local ...
Basics

Cluster name: rabbit@rabbitmq-cluster-0.rabbitmq-cluster.default.svc.cluster.local

Disk Nodes

rabbit@rabbitmq-cluster-0.rabbitmq-cluster.default.svc.cluster.local
rabbit@rabbitmq-cluster-1.rabbitmq-cluster.default.svc.cluster.local
rabbit@rabbitmq-cluster-2.rabbitmq-cluster.default.svc.cluster.local

Running Nodes

rabbit@rabbitmq-cluster-0.rabbitmq-cluster.default.svc.cluster.local
rabbit@rabbitmq-cluster-1.rabbitmq-cluster.default.svc.cluster.local
rabbit@rabbitmq-cluster-2.rabbitmq-cluster.default.svc.cluster.local

Versions

rabbit@rabbitmq-cluster-0.rabbitmq-cluster.default.svc.cluster.local: RabbitMQ 3.8.3 on Erlang 22.3.4.1
rabbit@rabbitmq-cluster-1.rabbitmq-cluster.default.svc.cluster.local: RabbitMQ 3.8.3 on Erlang 22.3.4.1
rabbit@rabbitmq-cluster-2.rabbitmq-cluster.default.svc.cluster.local: RabbitMQ 3.8.3 on Erlang 22.3.4.1

Alarms

(none)

Network Partitions

(none)

Listeners

Node: rabbit@rabbitmq-cluster-0.rabbitmq-cluster.default.svc.cluster.local, interface: [::], port: 25672, protocol: clustering, purpose: inter-node and CLI tool communication
Node: rabbit@rabbitmq-cluster-0.rabbitmq-cluster.default.svc.cluster.local, interface: [::], port: 5672, protocol: amqp, purpose: AMQP 0-9-1 and AMQP 1.0
Node: rabbit@rabbitmq-cluster-0.rabbitmq-cluster.default.svc.cluster.local, interface: [::], port: 15672, protocol: http, purpose: HTTP API
Node: rabbit@rabbitmq-cluster-1.rabbitmq-cluster.default.svc.cluster.local, interface: [::], port: 25672, protocol: clustering, purpose: inter-node and CLI tool communication
Node: rabbit@rabbitmq-cluster-1.rabbitmq-cluster.default.svc.cluster.local, interface: [::], port: 5672, protocol: amqp, purpose: AMQP 0-9-1 and AMQP 1.0
Node: rabbit@rabbitmq-cluster-1.rabbitmq-cluster.default.svc.cluster.local, interface: [::], port: 15672, protocol: http, purpose: HTTP API
Node: rabbit@rabbitmq-cluster-2.rabbitmq-cluster.default.svc.cluster.local, interface: [::], port: 25672, protocol: clustering, purpose: inter-node and CLI tool communication
Node: rabbit@rabbitmq-cluster-2.rabbitmq-cluster.default.svc.cluster.local, interface: [::], port: 5672, protocol: amqp, purpose: AMQP 0-9-1 and AMQP 1.0
Node: rabbit@rabbitmq-cluster-2.rabbitmq-cluster.default.svc.cluster.local, interface: [::], port: 15672, protocol: http, purpose: HTTP API

Feature flags

Flag: drop_unroutable_metric, state: enabled
Flag: empty_basic_get_metric, state: enabled
Flag: implicit_default_bindings, state: enabled
Flag: quorum_queue, state: enabled
Flag: virtual_host_metadata, state: enabled

登录web管理端口访问