k8s部署solrcloud+zookeeper集群

安装zookeeper

zk的yaml模板

apiVersion: v1
kind: Service
metadata:
  name: zookeeper-headless
  labels:
    app: zookeeper-headless
spec:
  ports:
  - name: client
    port: 2181
    targetPort: client
    protocol: TCP
  - name: server
    port: 2888
    targetPort: server
    protocol: TCP
  - name: election
    port: 3888
    targetPort: election
    protocol: TCP           
  selector:
    app: zk-cluster
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: zookeeper-cm
data:
  zkGenConfig.sh: |
    #!/usr/bin/env bash
    ZK_DATA_DIR=${ZK_DATA_DIR:-"/zookeeper/data"}
    ZK_DATA_LOG_DIR=${ZK_DATA_LOG_DIR:-"/zookeeper/datalog"}
    ZK_CONF_DIR=${ZK_CONF_DIR:-"/conf"}
    ZK_CONFIG_FILE="$ZK_CONF_DIR/zoo.cfg"
    ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181}
    ZK_SERVER_PORT=${ZK_SERVER_PORT:-2888}
    ZK_ELECTION_PORT=${ZK_ELECTION_PORT:-3888}
    ZK_TICK_TIME=${ZK_TICK_TIME:-2000}
    ZK_INIT_LIMIT=${ZK_INIT_LIMIT:-10}
    ZK_SYNC_LIMIT=${ZK_SYNC_LIMIT:-5}
    ZK_MAX_CLIENT_CNXNS=${ZK_MAX_CLIENT_CNXNS:-60}
    ID_FILE="$ZK_DATA_DIR/myid"    
    ZK_MIN_SESSION_TIMEOUT=${ZK_MIN_SESSION_TIMEOUT:- $((ZK_TICK_TIME*2))}
    ZK_MAX_SESSION_TIMEOUT=${ZK_MAX_SESSION_TIMEOUT:- $((ZK_TICK_TIME*20))}
    HOST=`hostname -s`
    DOMAIN=`hostname -d`
    function print_servers() {
      for (( i=1; i<=$ZK_REPLICAS; i++ ))
      do
        echo "server.$((i-1))=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT"
      done
    }
    function validate_env() {
        echo "Validating enviornment"
      if [ -z $ZK_REPLICAS ]; then
        echo "ZK_REPLICAS is a mandatory environment variable"
        exit 1
      fi
      if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
        NAME=${BASH_REMATCH[1]}
        ORD=${BASH_REMATCH[2]}
      else
        echo "Failed to extract ordinal from hostname $HOST"
        exit 1
      fi
      MY_ID=${ORD}
      echo "ZK_REPLICAS=$ZK_REPLICAS"
        echo "MY_ID=$MY_ID"
        echo "ZK_DATA_DIR=$ZK_DATA_DIR"
        echo "ZK_DATA_LOG_DIR=$ZK_DATA_LOG_DIR"
        echo "ZK_CLIENT_PORT=$ZK_CLIENT_PORT"
        echo "ZK_SERVER_PORT=$ZK_SERVER_PORT"
        echo "ZK_ELECTION_PORT=$ZK_ELECTION_PORT"
        echo "ZK_TICK_TIME=$ZK_TICK_TIME"
        echo "ZK_INIT_LIMIT=$ZK_INIT_LIMIT"
        echo "ZK_SYNC_LIMIT=$ZK_SYNC_LIMIT"
        echo "ZK_MAX_CLIENT_CNXNS=$ZK_MAX_CLIENT_CNXNS"
        echo "ZK_MIN_SESSION_TIMEOUT=$ZK_MIN_SESSION_TIMEOUT"
        echo "ZK_MAX_SESSION_TIMEOUT=$ZK_MAX_SESSION_TIMEOUT"
        echo "ENSEMBLE"
        print_servers
        echo "Enviorment validation successful"
    }
    function create_config() {
      rm -f $ZK_CONFIG_FILE
        echo "Creating ZooKeeper configuration"
        echo "#This file was autogenerated by zk DO NOT EDIT" >> $ZK_CONFIG_FILE
      echo "clientPort=$ZK_CLIENT_PORT" >> $ZK_CONFIG_FILE
        echo "dataDir=$ZK_DATA_DIR" >> $ZK_CONFIG_FILE
        echo "dataLogDir=$ZK_DATA_LOG_DIR" >> $ZK_CONFIG_FILE
        echo "tickTime=$ZK_TICK_TIME" >> $ZK_CONFIG_FILE
        echo "initLimit=$ZK_INIT_LIMIT" >> $ZK_CONFIG_FILE
        echo "syncLimit=$ZK_SYNC_LIMIT" >> $ZK_CONFIG_FILE
        echo "maxClientCnxns=$ZK_MAX_CLIENT_CNXNS" >> $ZK_CONFIG_FILE
        echo "minSessionTimeout=$ZK_MIN_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
        echo "maxSessionTimeout=$ZK_MAX_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
        if [ $ZK_REPLICAS -gt 1 ]; then 
          print_servers >> $ZK_CONFIG_FILE
        fi
        echo $MY_ID > $ID_FILE        
        echo "Wrote ZooKeeper configuration file to $ZK_CONFIG_FILE"
    }
    validate_env && create_config
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zk-cluster
spec:
  serviceName: zookeeper-headless
  replicas: 3
  template:
    metadata:
      labels:
        app: zk-cluster
    spec:
      containers:
        - command:
            - sh
            - '-c'
            - zkGenConfig.sh && exec zkServer.sh start-foreground          
          env:
            - name: ZK_REPLICAS
              value: "3"
            - name: ZK_CLIENT_PORT
              value: "2181"
            - name: ZK_SERVER_PORT
              value: "2888"
            - name: ZK_ELECTION_PORT
              value: "3888"
            - name: ZK_TICK_TIME
              value: "2000"          
            - name: ZK_INIT_LIMIT
              value: "5"
            - name: ZK_SYNC_LIMIT
              value: "10"
            - name: ZK_MAX_CLIENT_CNXNS
              value: "1024"
            - name: ZK_MAX_SESSION_TIMEOUT
              value: "40000"
            - name: ZK_MIN_SESSION_TIMEOUT
              value: "4000"
          name: zk-cluster
          imagePullPolicy: IfNotPresent
          image: zookeeper:3.4.9
          ports:
          - name: client
            containerPort: 2181
          - name: election
            containerPort: 3888
          - name: server
            containerPort: 2888                                      
          volumeMounts:
              - mountPath: /zookeeper
                name: zkdata
              - mountPath: /zookeeper-3.4.9/bin/zkGenConfig.sh
                name: zookeeper-cm
                subPath: zkGenConfig.sh
      volumes: 
      - name: zookeeper-cm
        configMap:
          name: zookeeper-cm
          defaultMode: 0655
          optional: false                      
  selector:
    matchLabels:
      app: zk-cluster
  updateStrategy:
    type: RollingUpdate
  # 使用storageClass做数据存储      
  volumeClaimTemplates:
    - metadata:
        name: zkdata
      spec:
        accessModes:
        - ReadWriteOnce
        storageClassName: "nfs-storage"
        resources:
          requests:
            storage: 10Gi

创建集群

# kubectl apply -f zookeeper-cluster.yaml

安装solr

solr 下载地址：
http://archive.apache.org/dist/lucene/solr/4.8.0/solr-4.8.0.tgz

构建docker镜像

# cat Dockerfile 
FROM tomcat:7.0.107 

ADD solr /usr/local/tomcat/webapps/solr/
# 添加了IK分词，不用的话可以不添加
COPY IKAnalyzer2012FF_u1.jar /usr/local/tomcat/webapps/solr/WEB-INF/lib/

RUN sed -i  '2c  JAVA_OPTS="$SOLR_JAVA_MEM -Xss1024K -XX:PermSize=128m -XX:MaxPermSize=256m -Dsolr.solr.home=$SOLR_HOME -Dhost=$SOLR_HOST -Dhost.port=$SOLR_PORT -Dhost.context=solr -DzkHost=$ZK_HOST"' /usr/local/tomcat/bin/catalina.sh
RUN mkdir -p /software/solrhome/

CMD /usr/local/tomcat/bin/catalina.sh run

solr的yaml模板

apiVersion: "v1"
kind: "Service"
metadata:
  name: "solrcloud-headless"
  labels:
    app: solrcloud-headless
spec:
  ports:
  - name: solr-client
    port: 8080
    targetPort: solr-client
    protocol: TCP
  selector:
    app: solrcloud
---
apiVersion: "v1"
kind: "ConfigMap"
metadata:
  name: "solrcloud-cm"
data:
  solr.xml: |
    <?xml version="1.0" encoding="UTF-8" ?>
    <solr >
      <solrcloud>
        <str name="host">${host:}</str>
        <int name="hostPort">${host.port:8983}</int>
        <str name="hostContext">${host.context:solr}</str>
        <int name="zkClientTimeout">${zkClientTimeout:30000}</int>
        <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
      </solrcloud>
      <shardHandlerFactory name="shardHandlerFactory"
        class="HttpShardHandlerFactory">
        <int name="socketTimeout">${socketTimeout:0}</int>
        <int name="connTimeout">${connTimeout:0}</int>
      </shardHandlerFactory>
    </solr>
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: solrcloud
spec:
  serviceName: solrcloud-headless
  replicas: 3
  template:
    metadata:
      labels:
        app: solrcloud
    spec:
      initContainers:
        - name: "cp-solr-xml"
          image: busybox:latest
          command: ['sh', '-c', 'cp /tmp/solr.xml /tmp-config/solr.xml']
          volumeMounts:
          - name: "solr-xml"
            mountPath: "/tmp"
          - name: "solrhome"
            mountPath: "/tmp-config"
      containers:
        - name: solr
          image: solrcloud:4.8.0
          imagePullPolicy: IfNotPresent
          ports:
          - containerPort: 8080
            name: solr-client
          env:
            - name: "SOLR_JAVA_MEM"
              value: "-Xms2g -Xmx2g"
            - name: "SOLR_HOME"
              value: "/software/solrhome/"
            - name: "SOLR_PORT"
              value: "8080"
            - name: "SOLR_HOST"
              value: "$(hostname -s).$(hostname -d)"   
              # 指定zookeeper集群的svc         
            - name: "ZK_HOST"
              value: "zookeeper-headless:2181"
          volumeMounts:
            - name: solrhome
              mountPath: /software/solrhome/
      volumes:
      - name: solr-xml
        configMap:
          name: solrcloud-cm
          items:
          - key: solr.xml
            path: solr.xml
  selector:
    matchLabels:
      app: solrcloud
  volumeClaimTemplates:
    - metadata:
        name: solrhome
      spec:
        accessModes:
        - ReadWriteOnce
        storageClassName: "nfs-storage"
        resources:
          requests:
            storage: 10Gi

创建集群

# kubectl apply -f solr-cluster.yam

创建solrcloud

先推送zookeeper配置文件，执行脚本需要配置java命令，然后使用solr包的推送

# ./solr-4.8.0/example/scripts/cloud-scripts/zkcli.sh  zk-cluster-0.zookeeper-headless:2181,zk-cluster-1.zookeeper-headless:2181,zk-cluster-2.zookeeper-headless:2181 cmd upconfig -confdir ./solr-4.8.0/example/example-schemaless/solr/collection1/conf/ -confname collection1

然后通过url访问solr创建

# curl "http://solrcloud-headless.my-tst.svc.comall.bj.public:8080/solr/admin/collections?action=CREATE&name=collection1&numShards=1&replicationFactor=3&collection.configName=collection1"