跳至主要內容

RabbitMQ-Cluster-Operator部署RabbitMQ集群

大约 4 分钟约 1139 字

RabbitMQ-Cluster-Operator部署RabbitMQ集群

0 参考资料

  • https://artifacthub.io/packages/helm/bitnami/rabbitmq-cluster-operator
  • https://github.com/rabbitmq/cluster-operator
  • https://www.rabbitmq.com/kubernetes/operator/operator-overview.html
  • https://www.rabbitmq.com/production-checklist.html
  • https://github.com/rabbitmq/cluster-operator/tree/main/docs/examples/
  • https://www.rabbitmq.com/kubernetes/operator/operator-monitoring.html
  • https://www.rabbitmq.com/kubernetes/operator/using-operator.html#service-availability

1 部署RabbitMQ-Cluster-Operator

# 下载官方提供的部署operator的清单文件
mkdir rabbitmq-operator
cd rabbitmq-operator/
wget https://github.com/rabbitmq/cluster-operator/releases/latest/download/cluster-operator.yml

# 将清单文件中用到的镜像私有化到私有镜像仓库
docker pull rabbitmqoperator/cluster-operator:2.1.0
docker tag rabbitmqoperator/cluster-operator:2.1.0 harbor.leadchina.cn/rabbitmqoperator/cluster-operator:2.1.0
docker push harbor.leadchina.cn/rabbitmqoperator/cluster-operator:2.1.0
# 修改operator的资源清单文件使用私有镜像仓库
# 应用资源清单文件部署operator
kubectl create -f cluster-operator.yml

2 使用RabbitMQ-Cluster-Opertor部署RabbitMQ集群

# 多参考官方的operator文档以及example
# rabbitmq-cluster-operator测试样例
https://github.com/rabbitmq/cluster-operator/tree/main/docs/examples/ 

# 将组建rabbitmq集群需要的镜像私有化放到私有仓库中
docker pull rabbitmq:3.10.2-management
docker tag rabbitmq:3.10.2-management harbor.leadchina.cn/rabbitmq/rabbitmq:3.10.2-management
docker push harbor.leadchina.cn/rabbitmq/rabbitmq:3.10.2-management

# 以下是可以用到生产环境的rabbitmq集群清单示例
# RabbitmqCluster类型的资源会被rabbitmq-cluster-operator监测并进行rabbtimq集群组建，同时也会创建Statefullset、Headless-service、以及外部访问的service
[root@salt-master-50 ~/rabbitmq-operator]# cat helloworld.yaml 
apiVersion: rabbitmq.com/v1beta1
kind: RabbitmqCluster
metadata:
  name: mq1
  namespace: default
  labels:
    app: rabbitmq
spec:
  image: harbor.leadchina.cn/rabbitmq/rabbitmq:3.10.2-management
  replicas: 5
  resources:
    requests:
      cpu: 2
      memory: 4Gi
    limits:
      cpu: 2
      memory: 4Gi
  service:
    type: NodePort
  affinity:
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          labelSelector:
            matchExpressions:
              - key: app.kubernetes.io/name
                operator: In
                values:
                - pod-anti-affinity
          topologyKey: kubernetes.io/hostname
  persistence:
    storageClassName: csi-rbd-sc
    storage: "5Gi"
  override:
    statefulSet:
      spec:
        template:
          spec:
            containers:
              - name: rabbitmq
                volumeMounts:
                  - mountPath: /var/lib/rabbitmq/quorum-segments
                    name: quorum-segments
                  - mountPath: /var/lib/rabbitmq/quorum-wal
                    name: quorum-wal
#            topologySpreadConstraints:
#            - maxSkew: 1
#              topologyKey: "topology.kubernetes.io/hostname"
#              whenUnsatisfiable: DoNotSchedule
#              labelSelector:
#                matchLabels:
#                  app.kubernetes.io/name: hello-world
        volumeClaimTemplates:
        - apiVersion: v1
          kind: PersistentVolumeClaim
          metadata:
            name: persistence
            namespace: default
          spec:
            accessModes:
              - ReadWriteOnce
            resources:
              requests:
                storage: 10Gi
            volumeMode: Filesystem
            storageClassName: csi-rbd-sc
        - apiVersion: v1
          kind: PersistentVolumeClaim
          metadata:
            name: quorum-wal
            namespace: default
          spec:
            accessModes:
              - ReadWriteOnce
            resources:
              requests:
                storage: 10Gi
            volumeMode: Filesystem
            storageClassName: csi-rbd-sc
        - apiVersion: v1
          kind: PersistentVolumeClaim
          metadata:
            name: quorum-segments
            namespace: default
          spec:
            accessModes:
              - ReadWriteOnce
            resources:
              requests:
                storage: 10Gi
            volumeMode: Filesystem
            storageClassName: csi-rbd-sc
  rabbitmq:
    envConfig: |
      RABBITMQ_LOGS=""
      RABBITMQ_QUORUM_DIR=/var/lib/rabbitmq/quorum-segments
    additionalPlugins:
      - rabbitmq_top
      - rabbitmq_shovel
#    advancedConfig: |
#      [
#          {ra, [
#              {wal_data_dir, '/var/lib/rabbitmq/quorum-wal'}
#          ]}
#      ].
    additionalConfig: |
      log.console = true
      log.console.level = debug
      log.console.formatter = json
      log.console.formatter.json.field_map = verbosity:v time msg domain file line pid level:-
      log.console.formatter.json.verbosity_map = debug:7 info:6 notice:5 warning:4 error:3 critical:2 alert:1 emergency:0
      log.console.formatter.time_format = epoch_usecs
      default_user=guest
      default_pass=guest
      cluster_partition_handling = pause_minority
      vm_memory_high_watermark_paging_ratio = 0.99
      disk_free_limit.relative = 1.5
      collect_statistics_interval = 10000


---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: production-ready-rabbitmq
spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: testmq
[root@salt-master-50 ~/rabbitmq-operator]# kubectl create -f helloworld.yaml

4 benchmark测试

# 测试前必须要先登录控制台创建测试用户，不能使用自带的guest用户测试
# 使用下面命令获取rabbitmq用户名和密码
NAMESPACE='default'
INSTANCE='mq1'
username=`kubectl -n $NAMESPACE get secret ${INSTANCE}-default-user -o jsonpath="{.data.username}" | base64 --decode`
password=`kubectl -n $NAMESPACE get secret ${INSTANCE}-default-user -o jsonpath="{.data.password}" | base64 --decode`
echo "user: ${username}, password: ${password}"
# 使用上面打印的用户和密码，然后再查看svc的nodeport在浏览器登录(觉得麻烦就直接使用下面命令创建)
# 创建用户并授予权限(登录到集群中的任意一个pod中使用下面命令创建)
rabbitmqctl add_user test Qwerty123456
rabbitmqctl set_user_tags test administrator
rabbitmqctl set_permissions -p / test ".*" ".*" ".*"

# 使用perf-test测试工具测试rabbitmq是否正常能使用
docker pull pivotalrabbitmq/perf-test
docker tag pivotalrabbitmq/perf-test:latest harbor.leadchina.cn/rabbitmq/perf-test:latest
docker push harbor.leadchina.cn/rabbitmq/perf-test:latest
# 压测脚本
[root@salt-master-50 ~/rabbitmq-operator]# cat benchmark-test.sh 
#!/bin/bash

instance=mq1
username='test'
password='Qwerty123456'
service=${instance}
kubectl get pod perf-test |grep perf-test > /dev/null 2>&1 && kubectl delete pod perf-test
kubectl run perf-test --image=harbor.leadchina.cn/rabbitmq/perf-test:latest -- --uri "amqp://${username}:${password}@${service}"
sleep 30
kubectl logs -f perf-test

# 执行压测操作
[root@salt-master-50 ~/rabbitmq-operator]# ./benchmark-test.sh

5 配置rabbitmq监控（prometheus）

# 创建rabbitmq集群专用的servicemonitor，可以根据自身需求调整
# 官方servicemonitor配置下载链接https://raw.githubusercontent.com/rabbitmq/cluster-operator/main/observability/prometheus/monitors/rabbitmq-servicemonitor.yml
[root@salt-master-50 ~/rabbitmq-operator]# cat rabbitmq-servicemonitor.yml 
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: rabbitmq
  labels:
    prometheus: service
  # If labels are defined in spec.serviceMonitorSelector.matchLabels of your deployed Prometheus object, make sure to include them here.
spec:
  endpoints:
  - port: prometheus
    scheme: http
    interval: 15s
    scrapeTimeout: 14s
  - port: prometheus-tls
    scheme: https
    interval: 15s
    scrapeTimeout: 14s
    tlsConfig:
      insecureSkipVerify: true # set to false and uncomment lines below to enable tls verification
        # ca:
        #   secret:
        #     key: ca.crt
        #     name: tls-secret # name of the secret containing the CA cert which signed the RabbitMQ Prometheus TLS cert
        # serverName: '*.RABBITMQ-INSTANCE-NAME.NAMESPACE.svc.cluster.local'
  - port: prometheus
    scheme: http
    path: /metrics/detailed
    params:
      family:
        - queue_coarse_metrics
        - queue_metrics
    interval: 15s
    scrapeTimeout: 14s
  - port: prometheus-tls
    scheme: https
    path: /metrics/detailed
    params:
      family:
        - queue_coarse_metrics
        - queue_metrics
    interval: 15s
    scrapeTimeout: 14s
    tlsConfig:
      insecureSkipVerify: true
  selector:
    matchLabels:
      app.kubernetes.io/component: rabbitmq
  namespaceSelector:
    any: true
[root@salt-master-50 ~/rabbitmq-operator]# 
# 配置grafana dashboard， 可以直接到官网搜 https://grafana.com/grafana/dashboards/10991-rabbitmq-overview/

6 修改rabbitmq集群节点类型

# RabbitMQ集群节点有两种类型，ram和disc
# disc可以保证队列及交换机数据一致性，但是性能一般，而ram类型的节点性能会比较好，如果集群中有多个节点，可以考虑将部分disc类型的节点转成ram类型
# 具体转换操作如下
# 在要修改的节点上操作
rabbitmqctl stop_app
# 等待节点完全停止后
rabbitmqctl change_cluster_node_type ram
# 启动应用
rabbitmqctl start_app
# 再次查看节点类型
rabbitmqctl cluster_status