1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

#修改文件描述符数量
grep "* - nofile 512000" /etc/security/limits.conf || echo  "* - nofile 512000"  >> /etc/security/limits.conf  

#修改最大打开进程数数量
grep "work - nproc unlimited" /etc/security/limits.conf || echo "elasticsearch - nproc unlimited"   >> /etc/security/limits.conf  
#配合es mem lock，centos6无须添加
grep "* soft memlock unlimited" /etc/security/limits.conf || echo "* soft memlock unlimited"   >> /etc/security/limits.conf  

#配合es mem lock，centos6无须添加
grep "* hard memlock unlimited" /etc/security/limits.conf || echo "* hard memlock unlimited"   >> /etc/security/limits.conf  

#修改系统文件描述符
grep "fs.file-max = 1024000" /etc/sysctl.conf || echo "fs.file-max = 1024000"  >> /etc/sysctl.conf 

#修改程序最大管理的vm
grep "vm.max_map_count = 262144" /etc/sysctl.conf || echo "vm.max_map_count = 262144"  >>  /etc/sysctl.conf  

grep  "vm.min_free_kbytes = 2097152" /etc/sysctl.conf || echo "vm.min_free_kbytes = 2097152"  >>  /etc/sysctl.conf

grep  "vm.zone_reclaim_mode = 0" /etc/sysctl.conf || echo "vm.zone_reclaim_mode = 0"  >>  /etc/sysctl.conf

sysctl -p

swapoff -a   #关闭虚拟内存

1
2
3
4

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.12.0-linux-x86_64.tar.gz
tar -xf elasticsearch-7.12.0-linux-x86_64.tar.gz -C ~
cd ~/elasticsearch-7.12.0
./bin/elasticsearch  # 启动

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

$ curl 127.0.0.1:9200
{
  "name" : "k8s-master",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "sEn3TgEVSnW4kHpIAU1-5Q",
  "version" : {
    "number" : "7.12.0",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "78722783c38caa25a70982b5b042074cde5d3b3a",
    "build_date" : "2021-03-18T06:17:15.410153305Z",
    "build_snapshot" : false,
    "lucene_version" : "8.8.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

1
2

-Xms1g 
-Xmx1g

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

# 启动实例1
./bin/elasticsearch -E cluster.name=myes -Enode.name=node0 \
-E node.master=true -E node.data=false -E node.ingest=false \
-E network.host=127.0.0.1 \
-E http.port=9200 -E transport.tcp.port=9300 \
-E discovery.seed_hosts="127.0.0.1:9300","127.0.0.1:9301","127.0.0.1:9302" \
-E cluster.initial_master_nodes="node0" -d

# 启动实例2 
./bin/elasticsearch -E cluster.name=myes -E node.name=node1 \
-E node.master=true -E node.data=true -E node.ingest=false \
-E path.data=./data/data_node1 -E network.host=127.0.0.1 \
-E http.port=9201 -E transport.tcp.port=9301 \
-E discovery.seed_hosts="127.0.0.1:9300","127.0.0.1:9301","127.0.0.1:9302" \
-d


# 启动实例3 
./bin/elasticsearch -E cluster.name=myes -E node.name=node2 \
-E node.master=true -E node.data=true -E node.ingest=false \
-E path.data=./data/data_node2 -E network.host=127.0.0.1 \
-E http.port=9202 -E transport.tcp.port=9302 \
-E discovery.seed_hosts="127.0.0.1:9300","127.0.0.1:9301","127.0.0.1:9302" \
-d

1
2
3
4

$ curl 127.0.0.1:9200/_cat/nodes
127.0.0.1 42 49 58 4.45 2.09 1.37 lmr        * node0
127.0.0.1 42 49 54 4.45 2.09 1.37 cdfhlmrstw - node1
127.0.0.1 42 49 45 4.45 2.09 1.37 cdfhlmrstw - node2

1
2
3
4

wget https://artifacts.elastic.co/downloads/kibana/kibana-7.12.0-linux-x86_64.tar.gz
tar -xf kibana-7.12.0-linux-x86_64.tar.gz
cd kibana-7.12.0-linux-x86_64

1
2
3
4
5

# 将kibana改成中文
vim config/kibana.yml
     i18n.locale: "zh-CN"   ## 最后一行
./bin/kibana

1
2

wget https://artifacts.elastic.co/downloads/logstash/logstash-7.12.0-linux-x86_64.tar.gz

1
2
3
4
5

wget https://github.com/lmenezes/cerebro/releases/download/v0.9.3/cerebro-0.9.3.tgz
tar -xf cerebro-0.9.3.tgz 
cd cerebro-0.9.3 
./bin/cerebro   # 启动

1
2

 wget http://files.grouplens.org/datasets/movielens/ml-20m.zip
 unzip ml-20m.zip

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

input {
  file {
    path => "/home/work/logs/ml-20m/movies.csv"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}
filter {
  csv {
    separator => ","
    columns => ["id","content","genre"]
  }

  mutate {
    split => { "genre" => "|" }
    remove_field => ["path", "host","@timestamp","message"]
  }

  mutate {

    split => ["content", "("]
    add_field => { "title" => "%{[content][0]}"}
    add_field => { "year" => "%{[content][1]}"}
  }

  mutate {
    convert => {
      "year" => "integer"
    }
    strip => ["title"]
    remove_field => ["path", "host","@timestamp","message","content"]
  }
}
output {
   elasticsearch {
     hosts => "http://localhost:9200"
     index => "movies"
     document_id => "%{id}"
   }
  stdout {}
}

1
2
3
4
5
6
7
8

xpack.security.http.ssl:
  enabled: false 【开启还是关闭】
  verification_mode: certificate【如下】
   【full：它验证所提供的证书是否由受信任的权威机构(CA)签名，并验证服务器的主机名(或IP地址)是否与证书中识别的名称匹配。】
   【certificate：它验证所提供的证书是否由受信任的机构(CA)签名，但不执行任何主机名验证。】
   【none：它不执行服务器证书的验证。】
  truststore.path: certs/elastic-certificates.p12 【信任存储库文件的存放位置】
  keystore.path: certs/elastic-certificates.p12【密钥存储库文件的存放位置】

1
2
3
4
5
6
7
8

xpack.security.transport.ssl:
  enabled: true【开启还是关闭】
  verification_mode: certificate【如下】
   【full：它验证所提供的证书是否由受信任的权威机构(CA)签名，并验证服务器的主机名(或IP地址)是否与证书中识别的名称匹配。】
   【certificate：它验证所提供的证书是否由受信任的机构(CA)签名，但不执行任何主机名验证。】
   【none：它不执行服务器证书的验证。】
  keystore.path: certs/elastic-certificates.p12【信任存储库文件的存放位置】
  truststore.path: certs/elastic-certificates.p12【密钥存储库文件的存放位置】

1

mkdir -p ~/data/elasticsearch-8.2.3/config/security

1
2
3
4
5
6
7

./bin/elasticsearch-certutil ca --ca-dn \
"CN=Elastic Certificate Tool Autogenerated CA" \
--days 3650 \
--out "./config/security/elastic-certificates.p12" \
--silent
Enter password for elastic-certificates.p12 : 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

$ vim ./config/security/instances.yml
instances:
  - name: "CN=es-node1.kiosk007.top"
    ip: 
      - "127.0.0.1"
    filename: "es-node1"
  - name: "CN=es-node2.kiosk007.top"
    ip:
      - "127.0.0.1"
    filename: "es-node2"
  - name: "CN=es-node3.kiosk007.top"
    ip: 
      - "127.0.0.1"
    filename: "es-node3"

# 生成证书，指定CA，配置证书密码为 transpasswd
$ ./bin/elasticsearch-certutil cert --silent \
 --in "./config/security/instances.yml" \
 --out "./config/security/certs.zip" \
 --ca "./config/security/elastic-certificates.p12" \
 --pass "transpasswd" --ca-pass "capasswd"

# 解压
$ unzip certs.zip
$ tree
security/
├── certs.zip
├── elastic-certificates.p12
├── es-node1
│   └── es-node1.p12
├── es-node2
│   └── es-node2.p12
├── es-node3
│   └── es-node3.p12
└── instances.yml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

# 交互式生成 http 证书 
$ ./bin/elasticsearch-certutil http --silent
## Elasticsearch HTTP Certificate Utility
## Do you wish to generate a Certificate Signing Request (CSR)?
# 是否生成 CSR，否（使用中央CA时选择 是）
Generate a CSR? [y/N]
## Do you have an existing Certificate Authority (CA) key-pair that you wish to use to sign your certificate?
# 是否使用已存在的 CA
Use an existing CA? [y/N]y
## What is the path to your CA?
# CA 路径
CA Path: ./config/security/elastic-certificates.p12
# CA 密码
Password for elastic-stack-ca.p12:
# 证书有效时间
For how long should your certificate be valid? [5y] 10y
# 是否为每个节点生成证书
Generate a certificate per node? [y/N]y

# 配置节点一信息
node #1 name: es-node1           
## Which hostnames will be used to connect to es-1?
es-node1.kiosk007.top

You entered the following hostnames.

 - es-node1.kiosk007.top
Is this correct [Y/n]
## Which IP addresses will be used to connect to es-node1?
127.0.0.1

You entered the following IP addresses.

 - 127.0.0.1
Is this correct [Y/n]
## Other certificate options
Key Name: es-node1
Subject DN: CN=es-node1
Key Size: 2048

Do you wish to change any of these options? [y/N]  # 选 N
Generate additional certificates? [Y/n] # 选 Y

# 配置节点二信息
node #2 name: es-node2
## Which hostnames will be used to connect to es-node2?
es-node2.kiosk007.top

You entered the following hostnames.

 - es-node2.kiosk007.top
Is this correct [Y/n]
## Which IP addresses will be used to connect to es-node2?
127.0.0.1

You entered the following IP addresses.

 - 127.0.0.1
Is this correct [Y/n]y
## Other certificate options
Key Name: es-node2
Subject DN: CN=es-node2
Key Size: 2048

Do you wish to change any of these options? [y/N] # 选 N
Generate additional certificates? [Y/n] # 选 Y

# 配置节点三信息
node #3 name: es-node3
## Which hostnames will be used to connect to es-node3?
es-node3.kiosk007.top

You entered the following hostnames.

 - es-node3.kiosk007.top
Is this correct [Y/n]
## Which IP addresses will be used to connect to es-node3?
127.0.0.1

You entered the following IP addresses.

 - 127.0.0.1
Is this correct [Y/n]
## Other certificate options
Key Name: es-node3
Subject DN: CN=es-node3
Key Size: 2048

Do you wish to change any of these options? [y/N]
Generate additional certificates? [Y/n]n
# 生成证书密码，此处使用 httppasswd
Provide a password for the "http.p12" file:  [<ENTER> for none]
Repeat password to confirm: 
# 生成证书路径
What filename should be used for the output zip file? [/home/ubuntu/data/elasticsearch-8.2.3/elasticsearch-ssl-http.zip] 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

$ tree
.
├── certs.zip
├── elastic-certificates.p12
├── elasticsearch
│   ├── es-node1
│   │   ├── README.txt
│   │   ├── http.p12
│   │   └── sample-elasticsearch.yml
│   ├── es-node2
│   │   ├── README.txt
│   │   ├── http.p12
│   │   └── sample-elasticsearch.yml
│   └── es-node3
│       ├── README.txt
│       ├── http.p12
│       └── sample-elasticsearch.yml
├── es-node1
│   └── es-node1.p12
├── es-node2
│   └── es-node2.p12
├── es-node3
│   └── es-node3.p12
├── instances.yml
└── kibana
    ├── README.txt
    ├── elasticsearch-ca.pem
    └── sample-kibana.yml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# 创建密钥库
$ ./bin/elasticsearch-keystore create
Created elasticsearch keystore in /home/ubuntu/data/elasticsearch-8.2.3/config/elasticsearch.keystore

# 存储传输层证书密码（此处为 transpoasswd）
$ ./bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password
$ ./bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password

# 存储http证书密码（此处为 httppasswd）
$ ./bin/elasticsearch-keystore add xpack.security.http.ssl.keystore.secure_password
$ ./bin/elasticsearch-keystore add xpack.security.http.ssl.truststore.secure_password

1

./bin/elasticsearch-reset-password -u elastic -E cluster.name=kiosk-es -Enode.name=es-node1 -E path.data=/home/ubuntu/data/elasticsearch-8.2.3/data/data1  -E path.logs=/home/ubuntu/data/elasticsearch-8.2.3/logs/log1 -E http.port=9200 -E transport.port=9300 -E network.host=127.0.0.1 -E discovery.seed_hosts="127.0.0.1:9300","127.0.0.1:9301" -E action.destructive_requires_name=true -E cluster.initial_master_nodes="es-node1","es-node2" -E xpack.security.enabled=true -E xpack.security.enrollment.enabled=true -E xpack.security.http.ssl.enabled=true -E xpack.security.http.ssl.verification_mode=certificate -E xpack.security.http.ssl.keystore.path=/home/ubuntu/data/elasticsearch-8.2.3/config/security/elasticsearch/es-node1/http.p12 -E xpack.security.http.ssl.truststore.path=/home/ubuntu/data/elasticsearch-8.2.3/config/security/elasticsearch/es-node1/http.p12 -E xpack.security.transport.ssl.enabled=true -E xpack.security.transport.ssl.verification_mode=certificate -E xpack.security.transport.ssl.keystore.path=/home/ubuntu/data/elasticsearch-8.2.3/config/security/es-node1/es-node1.p12 -E xpack.security.transport.ssl.truststore.path=/home/ubuntu/data/elasticsearch-8.2.3/config/security/es-node1/es-node1.p12

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# =================== System: Elasticsearch ===================
# The URLs of the Elasticsearch instances to use for all your queries.
elasticsearch.hosts: ["https://es-node1.kiosk007.top:9200","https://es-node2.kiosk007.top:9201"]

# If your Elasticsearch is protected with basic authentication, these settings provide
# the username and password that the Kibana server uses to perform maintenance on the Kibana
# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
# is proxied through the Kibana server.
elasticsearch.username: "kibana_system"
elasticsearch.password: "_pSr-80lrixxxxxxxxxxx"   # 上一步得到的密码

1

openssl pkcs12 -in elastic-certificates.p12 -clcerts -nokeys -chain -out elasic-certificates.pem

1
2
3
4
5

# Enables you to specify a path to the PEM file for the certificate
# authority for your Elasticsearch instance.
#elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ]
elasticsearch.ssl.certificateAuthorities: [ "/home/ubuntu/data/elasticsearch-8.2.3/config/security/elasic-certificates.pem" ]