Docker 容器中的 Elasticsearch - "failed to write in data directory" 简单重启后
Elasticsearch in Docker Container - "failed to write in data directory" after simple restart
几周前我创建了一个 elk 堆栈（elasticsearch、logstash、kibana）来更好地处理日志文件的负载。
一切都很完美。今天我将一些新模式调用到 logstash 中，出于某种原因，我通过 docker-compose down && docker-compose up -d.
重新启动
现在 elasticsearch 不再启动了。
root@xyz:/srv/elk# docker-compose logs elasticsearch
Attaching to elk_elasticsearch_1
elasticsearch_1 | [2017-07-01T07:34:36,859][INFO ][o.e.n.Node ] [lw-e01] initializing ...
elasticsearch_1 | [2017-07-01T07:34:36,999][INFO ][o.e.e.NodeEnvironment ] [lw-e01] using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/mapper/HDD-ELK)]], net usable_space [19.1gb], net total_space [49gb], spins? [possibly], types [ext4]
elasticsearch_1 | [2017-07-01T07:34:36,999][INFO ][o.e.e.NodeEnvironment ] [lw-e01] heap size [3.9gb], compressed ordinary object pointers [true]
elasticsearch_1 | [2017-07-01T07:34:37,635][WARN ][o.e.b.ElasticsearchUncaughtExceptionHandler] [lw-e01] uncaught exception in thread [main]
elasticsearch_1 | org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: Failed to created node environment
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:127) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:114) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:67) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:122) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.cli.Command.main(Command.java:88) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | Caused by: java.lang.IllegalStateException: Failed to created node environment
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:265) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.<init>(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:360) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:123) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | ... 6 more
elasticsearch_1 | Caused by: java.io.IOException: failed to write in data directory [/usr/share/elasticsearch/data/nodes/0/indices/a94kXbSER2CE97qdPhgVLA/_state] write permission is required
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.tryWriteTempFile(NodeEnvironment.java:1075) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.assertCanWrite(NodeEnvironment.java:1047) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:277) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:262) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.<init>(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:360) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:123) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | ... 6 more
elasticsearch_1 | Caused by: java.nio.file.FileAlreadyExistsException: /usr/share/elasticsearch/data/nodes/0/indices/a94kXbSER2CE97qdPhgVLA/_state/.es_temp_file
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:88) ~[?:?]
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) ~[?:?]
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) ~[?:?]
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) ~[?:?]
elasticsearch_1 | at java.nio.file.Files.newByteChannel(Files.java:361) ~[?:1.8.0_131]
elasticsearch_1 | at java.nio.file.Files.createFile(Files.java:632) ~[?:1.8.0_131]
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.tryWriteTempFile(NodeEnvironment.java:1072) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.assertCanWrite(NodeEnvironment.java:1047) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:277) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:262) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.<init>(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:360) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:123) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | ... 6 more
好的，它看起来像一个简单的权限，问题，但在 chown -R 1000.1000 elasticsearch/它崩溃（并设置其他所有权）之后也是如此。
设置：我为 docker-compose 项目设置了一个带有 LVM 的服务器。在docker-compose.yml我描述了三个服务。
version: '3'
services:
elasticsearch:
image: my/elasticsearch/image:5.4.0
volumes:
- ./elasticsearch/data:/usr/share/elasticsearch/data
- ./elasticsearch/config:/usr/share/elasticsearch/config
- /etc/localtime:/etc/localtime:ro
environment:
ES_JAVA_OPTS: "-Xmx4g -Xms1g"
ulimits:
memlock:
soft: -1
hard: -1
networks:
- nginx_net
logstash:
image: my/logstash/image:5.4.0
command: ["logstash", "-f", "/etc/logstash.conf"]
volumes:
- ./logstash.conf:/etc/logstash.conf:ro
- ./logstash.yml:/etc/logstash/logstash.yml:ro
- ./GeoDb/GeoLite2-City.mmdb:/GeoLite2-City.mmdb:ro
- ./patterns:/etc/logstash/patterns:ro
- /etc/localtime:/etc/localtime:ro
ports:
- "5044:5044"
environment:
LS_JAVA_OPTS: "-Xmx1g -Xms512m"
depends_on:
- elasticsearch
networks:
- nginx_net
kibana:
image: my/kibana/image:5.4.0
volumes:
- ./kibana/config/:/usr/share/kibana/config
- ./kibana/config/kibana.yml:/etc/kibana/kibana.yml
- /etc/localtime:/etc/localtime:ro
depends_on:
- elasticsearch
networks:
- nginx_net
networks:
nginx_net:
external: true
正如你所看到的，我没有直接使用官方图像，我也安装了（此刻）XPack，所有三个图像都是这样的
FROM elasticsearch:5.4.0
RUN bin/elasticsearch-plugin install x-pack --batch
我做的第二件不同的事情是我没有使用命名卷。那是因为我喜欢一个文件夹包含整个项目，也更适合我的 LVM 管理。
root@xyz:/srv/elk# ls -l
insgesamt 43488
-rw-r--r-- 1 root root 1514 Jul 1 09:34 docker-compose.yml
drwxr-xr-x 4 1000 1000 4096 Mai 18 17:43 elasticsearch
drwxr-xr-x 3 root root 4096 Mai 21 12:49 GeoDb
-rw-r--r-- 1 root root 25398754 Mai 21 12:49 GeoLite2-City.tar.gz
-rw-r--r-- 1 root root 19074950 Mai 21 12:03 GeoLiteCity.dat
drwxr-xr-x 3 root root 4096 Mai 14 16:20 kibana
-rw-r--r-- 1 root root 5523 Jul 1 09:02 logstash.conf
-rw-r--r-- 1 root root 4708 Jun 3 11:25 logstash.yml
drwx------ 2 root root 16384 Mai 17 23:40 lost+found
drwxr-xr-x 2 root root 4096 Jun 7 22:08 patterns
-rwxr-xr-x 1 root root 168 Mai 21 12:49 update-geoip.sh
root@xyz:/srv/elk# du -hs elasticsearch/
28G elasticsearch/
我读到有关 local-persist 之类的插件可以使用命名卷，还可以指定要将文件保存到的目录。但我也读到，docker 建议不要在生产中使用插件。
我会很高兴有任何想法/link。
OK 简单：运行（在我的例子中）docker-compose 项目的主文件夹中的 rm elasticsearch/data/nodes/0/indices/a94kXbSER2CE97qdPhgVLA/_state/.es_temp_file 帮助我再次启动 EL...
要找出确切的路径，请查看 java.nio.file.FileAlreadyExistsException
以上答案对我不起作用。它最终成为 Docker 的内存问题。提升 Docker 的内存分配解决了它。
几周前我创建了一个 elk 堆栈（elasticsearch、logstash、kibana）来更好地处理日志文件的负载。
一切都很完美。今天我将一些新模式调用到 logstash 中，出于某种原因，我通过 docker-compose down && docker-compose up -d.
现在 elasticsearch 不再启动了。
root@xyz:/srv/elk# docker-compose logs elasticsearch
Attaching to elk_elasticsearch_1
elasticsearch_1 | [2017-07-01T07:34:36,859][INFO ][o.e.n.Node ] [lw-e01] initializing ...
elasticsearch_1 | [2017-07-01T07:34:36,999][INFO ][o.e.e.NodeEnvironment ] [lw-e01] using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/mapper/HDD-ELK)]], net usable_space [19.1gb], net total_space [49gb], spins? [possibly], types [ext4]
elasticsearch_1 | [2017-07-01T07:34:36,999][INFO ][o.e.e.NodeEnvironment ] [lw-e01] heap size [3.9gb], compressed ordinary object pointers [true]
elasticsearch_1 | [2017-07-01T07:34:37,635][WARN ][o.e.b.ElasticsearchUncaughtExceptionHandler] [lw-e01] uncaught exception in thread [main]
elasticsearch_1 | org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: Failed to created node environment
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:127) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:114) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:67) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:122) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.cli.Command.main(Command.java:88) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:91) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:84) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | Caused by: java.lang.IllegalStateException: Failed to created node environment
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:265) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.<init>(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:360) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:123) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | ... 6 more
elasticsearch_1 | Caused by: java.io.IOException: failed to write in data directory [/usr/share/elasticsearch/data/nodes/0/indices/a94kXbSER2CE97qdPhgVLA/_state] write permission is required
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.tryWriteTempFile(NodeEnvironment.java:1075) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.assertCanWrite(NodeEnvironment.java:1047) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:277) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:262) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.<init>(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:360) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:123) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | ... 6 more
elasticsearch_1 | Caused by: java.nio.file.FileAlreadyExistsException: /usr/share/elasticsearch/data/nodes/0/indices/a94kXbSER2CE97qdPhgVLA/_state/.es_temp_file
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:88) ~[?:?]
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) ~[?:?]
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) ~[?:?]
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) ~[?:?]
elasticsearch_1 | at java.nio.file.Files.newByteChannel(Files.java:361) ~[?:1.8.0_131]
elasticsearch_1 | at java.nio.file.Files.createFile(Files.java:632) ~[?:1.8.0_131]
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.tryWriteTempFile(NodeEnvironment.java:1072) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.assertCanWrite(NodeEnvironment.java:1047) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:277) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:262) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.node.Node.<init>(Node.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.<init>(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:242) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:360) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:123) ~[elasticsearch-5.4.0.jar:5.4.0]
elasticsearch_1 | ... 6 more
好的，它看起来像一个简单的权限，问题，但在 chown -R 1000.1000 elasticsearch/它崩溃（并设置其他所有权）之后也是如此。
设置：我为 docker-compose 项目设置了一个带有 LVM 的服务器。在docker-compose.yml我描述了三个服务。
version: '3'
services:
elasticsearch:
image: my/elasticsearch/image:5.4.0
volumes:
- ./elasticsearch/data:/usr/share/elasticsearch/data
- ./elasticsearch/config:/usr/share/elasticsearch/config
- /etc/localtime:/etc/localtime:ro
environment:
ES_JAVA_OPTS: "-Xmx4g -Xms1g"
ulimits:
memlock:
soft: -1
hard: -1
networks:
- nginx_net
logstash:
image: my/logstash/image:5.4.0
command: ["logstash", "-f", "/etc/logstash.conf"]
volumes:
- ./logstash.conf:/etc/logstash.conf:ro
- ./logstash.yml:/etc/logstash/logstash.yml:ro
- ./GeoDb/GeoLite2-City.mmdb:/GeoLite2-City.mmdb:ro
- ./patterns:/etc/logstash/patterns:ro
- /etc/localtime:/etc/localtime:ro
ports:
- "5044:5044"
environment:
LS_JAVA_OPTS: "-Xmx1g -Xms512m"
depends_on:
- elasticsearch
networks:
- nginx_net
kibana:
image: my/kibana/image:5.4.0
volumes:
- ./kibana/config/:/usr/share/kibana/config
- ./kibana/config/kibana.yml:/etc/kibana/kibana.yml
- /etc/localtime:/etc/localtime:ro
depends_on:
- elasticsearch
networks:
- nginx_net
networks:
nginx_net:
external: true
正如你所看到的，我没有直接使用官方图像，我也安装了（此刻）XPack，所有三个图像都是这样的
FROM elasticsearch:5.4.0
RUN bin/elasticsearch-plugin install x-pack --batch
我做的第二件不同的事情是我没有使用命名卷。那是因为我喜欢一个文件夹包含整个项目，也更适合我的 LVM 管理。
root@xyz:/srv/elk# ls -l
insgesamt 43488
-rw-r--r-- 1 root root 1514 Jul 1 09:34 docker-compose.yml
drwxr-xr-x 4 1000 1000 4096 Mai 18 17:43 elasticsearch
drwxr-xr-x 3 root root 4096 Mai 21 12:49 GeoDb
-rw-r--r-- 1 root root 25398754 Mai 21 12:49 GeoLite2-City.tar.gz
-rw-r--r-- 1 root root 19074950 Mai 21 12:03 GeoLiteCity.dat
drwxr-xr-x 3 root root 4096 Mai 14 16:20 kibana
-rw-r--r-- 1 root root 5523 Jul 1 09:02 logstash.conf
-rw-r--r-- 1 root root 4708 Jun 3 11:25 logstash.yml
drwx------ 2 root root 16384 Mai 17 23:40 lost+found
drwxr-xr-x 2 root root 4096 Jun 7 22:08 patterns
-rwxr-xr-x 1 root root 168 Mai 21 12:49 update-geoip.sh
root@xyz:/srv/elk# du -hs elasticsearch/
28G elasticsearch/
我读到有关 local-persist 之类的插件可以使用命名卷，还可以指定要将文件保存到的目录。但我也读到，docker 建议不要在生产中使用插件。
我会很高兴有任何想法/link。
OK 简单：运行（在我的例子中）docker-compose 项目的主文件夹中的 rm elasticsearch/data/nodes/0/indices/a94kXbSER2CE97qdPhgVLA/_state/.es_temp_file 帮助我再次启动 EL...
要找出确切的路径，请查看 java.nio.file.FileAlreadyExistsException
以上答案对我不起作用。它最终成为 Docker 的内存问题。提升 Docker 的内存分配解决了它。