Last active
May 25, 2022 16:39
Revisions
-
mocobeta revised this gist
Jun 14, 2020 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -14,5 +14,5 @@ data: xpack.monitoring.collection.enabled: "false" xpack.monitoring.enabled: "false" xpack.security.enabled: "false" xpack.sql.enabled: "false" xpack.watcher.enabled: "false" -
mocobeta revised this gist
Jun 14, 2020 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -6,7 +6,7 @@ data: bootstrap.memory_lock: "false" # can't lock memory on GKE. instead, set "vm.swappiness=1" when initializing containers. (see the StatefulSet manifests.) http.port: "9200" cluster.initial_master_nodes: "elasticsearch-master-0, elasticsearch-master-1, elasticsearch-master-2" cluster.name: "my-es-cluster" discovery.seed_hosts: "elasticsearch-master" ES_JAVA_OPTS: -Xms1g -Xmx1g # for testing xpack.license.self_generated.type: "basic" -
mocobeta revised this gist
Jun 14, 2020 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -23,7 +23,7 @@ gcloud container images describe ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}: # # connect to GKE cluster # note: before executing, you have to create a GKE cluster named 'elasticsearch'. # gcloud container clusters get-credentials ${GKE_CLUSTER} --region asia-northeast1 --project ${PROJECT} kubectl get nodes -
mocobeta revised this gist
Jun 13, 2020 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -58,6 +58,6 @@ kubectl exec -it elasticsearch-data-0 -- curl "localhost:9200/_nodes/stats/indic # delete a data node kubectl delete pod elasticsearch-data-1 # verify auto recoverying watch kubectl get pods kubectl exec -it elasticsearch-data-0 -- curl localhost:9200/_cluster/health?pretty -
mocobeta revised this gist
Jun 13, 2020 . 1 changed file with 18 additions and 0 deletions.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -43,3 +43,21 @@ kubectl apply -f statefulset-data.yaml # verify if the cluster is properly created kubectl get nodes kubectl exec -it elasticsearch-data-0 -- curl localhost:9200/_cluster/health?pretty ##################################################### # commands for veryfing StatefulSet auto recovering # ##################################################### # index test data kubectl exec -it elasticsearch-data-0 -- curl -XPUT localhost:9200/testidx -HContent-type:application/json --data '{"settings": {"index": {"number_of_shards": 1, "number_of_replicas": 2}}}' for i in {1..100}; do kubectl exec elasticsearch-data-0 -- curl -s -XPOST localhost:9200/testidx/_doc -HContent-type:application/json --data '{"data":"test"}'; done kubectl exec -it elasticsearch-data-0 -- curl "localhost:9200/testidx/_search?size=0&pretty" kubectl exec -it elasticsearch-data-0 -- curl "localhost:9200/_nodes/stats/indices/docs?pretty" # delete a data node kubectl delete pod elasticsearch-data-1 # verify autl recoverying watch kubectl get pods kubectl exec -it elasticsearch-data-0 -- curl localhost:9200/_cluster/health?pretty -
mocobeta revised this gist
Jun 13, 2020 . 2 changed files with 2 additions and 2 deletions.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -42,7 +42,7 @@ spec: terminationGracePeriodSeconds: 300 containers: - name: elasticsearch image: asia.gcr.io/my-project-may31/my-elasticsearch-7:0.1.0 # FIXME resources: requests: cpu: "400m" # for testing This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -42,7 +42,7 @@ spec: terminationGracePeriodSeconds: 300 containers: - name: elasticsearch image: asia.gcr.io/my-project-may31/my-elasticsearch-7:0.1.0 # FIXME resources: requests: cpu: "400m" # for testing -
mocobeta revised this gist
Jun 13, 2020 . 1 changed file with 2 additions and 2 deletions.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -3,7 +3,7 @@ # export PROJECT=$(gcloud config get-value project) export REGISTRY_HOST=asia.gcr.io export CONTAINER_IMAGE=my-elasticsearch-7 export IMAGE_VERSION=0.1.0 export GKE_CLUSTER=elasticsearch @@ -19,7 +19,7 @@ gcloud auth configure-docker docker build -q --tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} -f Dockerfile-${IMAGE_VERSION} --no-cache . docker tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} docker push ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} gcloud container images describe ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} # # connect to GKE cluster -
mocobeta revised this gist
Jun 13, 2020 . 2 changed files with 2 additions and 2 deletions.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -42,7 +42,7 @@ spec: terminationGracePeriodSeconds: 300 containers: - name: elasticsearch image: asia.gcr.io/my-project-may31/my-es-7:0.1.0 # FIXME resources: requests: cpu: "400m" # for testing This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -42,7 +42,7 @@ spec: terminationGracePeriodSeconds: 300 containers: - name: elasticsearch image: asia.gcr.io/my-project-may31/my-es-7:0.1.0 # FIXME resources: requests: cpu: "400m" # for testing -
mocobeta revised this gist
Jun 13, 2020 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -19,7 +19,7 @@ gcloud auth configure-docker docker build -q --tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} -f Dockerfile-${IMAGE_VERSION} --no-cache . docker tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} docker push ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} gcloud container images describe ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE} # # connect to GKE cluster -
mocobeta revised this gist
Jun 13, 2020 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -18,7 +18,7 @@ gcloud auth configure-docker # docker build -q --tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} -f Dockerfile-${IMAGE_VERSION} --no-cache . docker tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} docker push ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} gcloud container images list --repository ${REGISTRY_HOST}/${PROJECT} # -
mocobeta revised this gist
Jun 13, 2020 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -16,7 +16,7 @@ gcloud auth configure-docker # # build and push docker image # docker build -q --tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} -f Dockerfile-${IMAGE_VERSION} --no-cache . docker tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} docker push ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE} gcloud container images list --repository ${REGISTRY_HOST}/${PROJECT} -
mocobeta revised this gist
Jun 13, 2020 . 1 changed file with 10 additions and 0 deletions.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,10 @@ FROM docker.elastic.co/elasticsearch/elasticsearch:7.7.0 ENV PATH /usr/share/elasticsearch/bin:$PATH # switch user to elasticsearch USER elasticsearch # install plugins RUN elasticsearch-plugin install analysis-kuromoji RUN elasticsearch-plugin install analysis-icu -
mocobeta revised this gist
Jun 13, 2020 . 2 changed files with 2 additions and 2 deletions.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -7,4 +7,4 @@ USER elasticsearch # install plugins RUN elasticsearch-plugin install analysis-kuromoji This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -16,7 +16,7 @@ gcloud auth configure-docker # # build and push docker image # docker build -q --tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} -f Dockerfile-0.1.0 --no-cache . docker tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} docker push ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE} gcloud container images list --repository ${REGISTRY_HOST}/${PROJECT} -
mocobeta revised this gist
Jun 13, 2020 . No changes.There are no files selected for viewing
-
mocobeta revised this gist
Jun 13, 2020 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -14,7 +14,7 @@ gcloud init gcloud auth configure-docker # # build and push docker image # docker build -q --tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} --no-cache . docker tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} -
mocobeta revised this gist
Jun 13, 2020 . 1 changed file with 45 additions and 0 deletions.There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,45 @@ # # variables # export PROJECT=$(gcloud config get-value project) export REGISTRY_HOST=asia.gcr.io export CONTAINER_IMAGE=my-es-7 export IMAGE_VERSION=0.1.0 export GKE_CLUSTER=elasticsearch # # authentication # gcloud init gcloud auth configure-docker # # build and push docker image image to container registry # docker build -q --tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} --no-cache . docker tag ${CONTAINER_IMAGE}:${IMAGE_VERSION} ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE}:${IMAGE_VERSION} docker push ${REGISTRY_HOST}/${PROJECT}/${CONTAINER_IMAGE} gcloud container images list --repository ${REGISTRY_HOST}/${PROJECT} # # connect to GKE cluster # note: before executing, you have to create a GKE cluster named 'elasticsearch' and an node pool that has at least 6 nodes. # gcloud container clusters get-credentials ${GKE_CLUSTER} --region asia-northeast1 --project ${PROJECT} kubectl get nodes # # apply manifests # kubectl apply -f configmap-common.yaml kubectl apply -f configmap-data.yaml kubectl apply -f configmap-master.yaml kubectl apply -f service-data.yaml kubectl apply -f service-master.yaml kubectl apply -f storageclass-ssd.yaml kubectl apply -f statefulset-master.yaml kubectl apply -f statefulset-data.yaml # verify if the cluster is properly created kubectl get nodes kubectl exec -it elasticsearch-data-0 -- curl localhost:9200/_cluster/health?pretty -
mocobeta created this gist
Jun 13, 2020 .There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,10 @@ FROM docker.elastic.co/elasticsearch/elasticsearch:7.7.0 ENV PATH /usr/share/elasticsearch/bin:$PATH # switch user to elasticsearch USER elasticsearch # install plugins RUN elasticsearch-plugin install analysis-kuromoji RUN elasticsearch-plugin install analysis-icu This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,18 @@ apiVersion: v1 kind: ConfigMap metadata: name: elasticsearch-common-config data: bootstrap.memory_lock: "false" # can't lock memory on GKE. instead, set "vm.swappiness=1" when initializing containers. (see the StatefulSet manifests.) http.port: "9200" cluster.initial_master_nodes: "elasticsearch-master-0, elasticsearch-master-1, elasticsearch-master-2" cluster.name: "my-es7-cluster" discovery.seed_hosts: "elasticsearch-master" ES_JAVA_OPTS: -Xms1g -Xmx1g # for testing xpack.license.self_generated.type: "basic" xpack.ml.enabled: "false" xpack.monitoring.collection.enabled: "false" xpack.monitoring.enabled: "false" xpack.security.enabled: "false" xpack.sql.enabled: "true" xpack.watcher.enabled: "false" This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,9 @@ apiVersion: v1 kind: ConfigMap metadata: name: elasticsearch-data-config data: node.data: "true" node.ingest: "false" node.master: "false" node.ml: "false" This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,9 @@ apiVersion: v1 kind: ConfigMap metadata: name: elasticsearch-master-config data: node.data: "false" node.ingest: "false" node.master: "true" node.ml: "false" This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,17 @@ apiVersion: v1 kind: Service metadata: name: elasticsearch-data spec: selector: app: elasticsearch-data type: ClusterIP ports: - protocol: TCP name: rest port: 9200 targetPort: 9200 - protocol: TCP name: nodes port: 9300 targetPort: 9300 This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,17 @@ apiVersion: v1 kind: Service metadata: name: elasticsearch-master spec: selector: app: elasticsearch-master type: ClusterIP ports: - protocol: TCP name: rest port: 9200 targetPort: 9200 - protocol: TCP name: nodes port: 9300 targetPort: 9300 This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,81 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: elasticsearch-data spec: selector: matchLabels: app: elasticsearch-data serviceName: elasticsearch-data replicas: 3 updateStrategy: type: RollingUpdate template: metadata: labels: app: elasticsearch-data spec: initContainers: - name: increase-the-ulimit image: busybox command: ["sh", "-c", "ulimit -n 65536"] securityContext: privileged: true - name: configure-swappiness image: busybox command: ["sh", "-c", "sysctl -w vm.swappiness=1"] securityContext: privileged: true - name: max-map-count image: busybox command: ["sh", "-c", "sysctl -w vm.max_map_count=262144"] securityContext: privileged: true - name: chown-mountpoint image: busybox command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"] securityContext: privileged: true volumeMounts: - name: es-data mountPath: /usr/share/elasticsearch/data terminationGracePeriodSeconds: 300 containers: - name: elasticsearch image: asia.gcr.io/my-project-may31/my-es-7:0.1.0 resources: requests: cpu: "400m" # for testing ports: - containerPort: 9200 name: rest #readinessProbe: # Note: disable the readiness probe only when cluster bootstrapping # exec: # command: # - "sh" # - "-c" # - "curl -s http://localhost:9200/ | grep 'cluster_uuid' | grep -v '_na_'" # check whether cluster_uuid is set # periodSeconds: 20 livenessProbe: httpGet: port: 9200 path: / initialDelaySeconds: 300 periodSeconds: 20 volumeMounts: - name: es-data mountPath: /usr/share/elasticsearch/data envFrom: - configMapRef: name: elasticsearch-common-config - configMapRef: name: elasticsearch-data-config volumeClaimTemplates: - metadata: name: es-data spec: storageClassName: ssd accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 100Gi This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,81 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: elasticsearch-master spec: selector: matchLabels: app: elasticsearch-master serviceName: elasticsearch-master replicas: 3 updateStrategy: type: RollingUpdate template: metadata: labels: app: elasticsearch-master spec: initContainers: - name: increase-the-ulimit image: busybox command: ["sh", "-c", "ulimit -n 65536"] securityContext: privileged: true - name: configure-swappiness image: busybox command: ["sh", "-c", "sysctl -w vm.swappiness=1"] securityContext: privileged: true - name: max-map-count image: busybox command: ["sh", "-c", "sysctl -w vm.max_map_count=262144"] securityContext: privileged: true - name: chown-mountpoint image: busybox command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"] securityContext: privileged: true volumeMounts: - name: es-data mountPath: /usr/share/elasticsearch/data terminationGracePeriodSeconds: 300 containers: - name: elasticsearch image: asia.gcr.io/my-project-may31/my-es-7:0.1.0 resources: requests: cpu: "400m" # for testing ports: - containerPort: 9200 name: rest #readinessProbe: # Note: disable the readiness probe only when cluster bootstrapping # exec: # command: # - "sh" # - "-c" # - "curl -s http://localhost:9200/ | grep 'cluster_uuid' | grep -v '_na_'" # check whether cluster_uuid is set # periodSeconds: 20 livenessProbe: httpGet: port: 9200 path: / initialDelaySeconds: 300 periodSeconds: 20 volumeMounts: - name: es-data mountPath: /usr/share/elasticsearch/data envFrom: - configMapRef: name: elasticsearch-common-config - configMapRef: name: elasticsearch-master-config volumeClaimTemplates: - metadata: name: es-data spec: storageClassName: ssd accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 50Gi This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ssd provisioner: kubernetes.io/gce-pd parameters: type: pd-ssd