1
Мой кластер elasticserach упал, когда я пытался создать индексный шаблон в Kibana. Ни один из API кота не работает, например. _cat/узлыКластер Elasticsearch ClusterStatsCollector отключен
В эластичных бревен мастер-узел Я вижу:
Caused by: org.elasticsearch.transport.ReceiveTimeoutTransportException: [data-3-vm-zone-1227643n][xx.xx.133.151:9300][cluster:monitor/nodes/stats[n]] request_id [9651533] timed out after [15000ms]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:908) ~[elasticsearch-5.2.0.jar:5.2.0]
... 4 more
[2017-02-15T06:38:40,046][ERROR][o.e.x.m.c.i.IndexRecoveryCollector] [master1-vm-zone-p1227651n] collector [index-recovery-collector] timed out when collecting data
[2017-02-15T06:38:50,047][ERROR][o.e.x.m.c.i.IndexStatsCollector] [master1-vm-zone-p1227651n] collector [index-stats-collector] timed out when collecting data
[2017-02-15T06:39:00,048][ERROR][o.e.x.m.c.i.IndicesStatsCollector] [master1-vm-zone-p1227651n] collector [indices-stats-collector] timed out when collecting data
[2017-02-15T06:39:10,048][ERROR][o.e.x.m.c.c.ClusterStatsCollector] [master1-vm-zone-p1227651n] collector [cluster-stats-collector] timed out when collecting data
[2017-02-15T06:39:30,152][ERROR][o.e.x.m.c.i.IndexRecoveryCollector] [master1-vm-zone-p1227651n] collector [index-recovery-collector] timed out when collecting data
[2017-02-15T06:39:31,884][DEBUG][o.e.a.a.c.n.s.TransportNodesStatsAction] [master1-vm-zone-p1227651n] failed to execute on node [FN8JwM_rS7WgSP0h9hXvMA]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [data-3-vm-zone-p1227643n][xx.xx.133.151:9300][cluster:monitor/nodes/stats[n]] request_id [9651829] timed out after [15000ms]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:908) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:527) [elasticsearch-5.2.0.jar:5.2.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_60]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_60]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_60]
[2017-02-15T06:39:31,884][DEBUG][o.e.a.a.c.n.s.TransportNodesStatsAction] [master1-vm-zone-p1227651n] failed to execute on node [yyaGz1WWQyyQXmstHpKSbw]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [data-2-vm-zone-p1227642n][xx.xx.133.150:9300][cluster:monitor/nodes/stats[n]] request_id [9651827] timed out after [15000ms]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:908) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:527) [elasticsearch-5.2.0.jar:5.2.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_60]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_60]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_60]
[2017-02-15T06:39:31,884][DEBUG][o.e.a.a.c.n.s.TransportNodesStatsAction] [master1-vm-zone-p1227651n] failed to execute on node [DEw888fjSMm0ODb3D7nMPg]
org.elasticsearch.transport.ReceiveTimeoutTransportException: [data-1-vm-zone-p1227641n][xx.xx.133.149:9300][cluster:monitor/nodes/stats[n]] request_id [9651828] timed out after [15000ms]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:908) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:527) [elasticsearch-5.2.0.jar:5.2.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_60]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_60]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_60]
[2017-02-15T06:39:31,885][WARN ][o.e.a.a.c.n.s.TransportNodesStatsAction] [master1-vm-zone-p1227651n] not accumulating exceptions, excluding exception from response
org.elasticsearch.action.FailedNodeException: Failed node [yyaGz1WWQyyQXmstHpKSbw]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.onFailure(TransportNodesAction.java:247) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.access$300(TransportNodesAction.java:160) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1.handleException(TransportNodesAction.java:219) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1024) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:907) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:527) [elasticsearch-5.2.0.jar:5.2.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_60]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_60]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_60]
Caused by: org.elasticsearch.transport.ReceiveTimeoutTransportException: [data-2-vm-zone-p1227642n][xx.xx.133.150:9300][cluster:monitor/nodes/stats[n]] request_id [9651827] timed out after [15000ms]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:908) ~[elasticsearch-5.2.0.jar:5.2.0]
... 4 more
[2017-02-15T06:39:31,886][WARN ][o.e.a.a.c.n.s.TransportNodesStatsAction] [master1-vm-zone-p1227651n] not accumulating exceptions, excluding exception from response
org.elasticsearch.action.FailedNodeException: Failed node [DEw888fjSMm0ODb3D7nMPg]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.onFailure(TransportNodesAction.java:247) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.access$300(TransportNodesAction.java:160) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1.handleException(TransportNodesAction.java:219) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1024) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:907) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:527) [elasticsearch-5.2.0.jar:5.2.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_60]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_60]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_60]
Caused by: org.elasticsearch.transport.ReceiveTimeoutTransportException: [data-1-vm-zone-p1227641n][xx.xx.133.149:9300][cluster:monitor/nodes/stats[n]] request_id [9651828] timed out after [15000ms]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:908) ~[elasticsearch-5.2.0.jar:5.2.0]
... 4 more
[2017-02-15T06:39:31,886][WARN ][o.e.a.a.c.n.s.TransportNodesStatsAction] [master1-vm-zone-p1227651n] not accumulating exceptions, excluding exception from response
org.elasticsearch.action.FailedNodeException: Failed node [FN8JwM_rS7WgSP0h9hXvMA]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.onFailure(TransportNodesAction.java:247) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.access$300(TransportNodesAction.java:160) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1.handleException(TransportNodesAction.java:219) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1024) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:907) [elasticsearch-5.2.0.jar:5.2.0]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:527) [elasticsearch-5.2.0.jar:5.2.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_60]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_60]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_60]
Caused by: org.elasticsearch.transport.ReceiveTimeoutTransportException: [data-3-vm-zone-p1227643n][xx.xx.133.151:9300][cluster:monitor/nodes/stats[n]] request_id [9651829] timed out after [15000ms]
at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:908) ~[elasticsearch-5.2.0.jar:5.2.0]
... 4 more
Любая идея, что может пошло не так?