snapshot parse json failure #269
Description
Hello,
having v1.1.0rc1 with --es.snapshots and getting:
elasticsearch_snapshot_stats_json_parse_failures 207
elasticsearch_snapshot_stats_total_scrapes 207
elasticsearch_snapshot_stats_up 1
Manual requests provide valid json:
$ curl -s localhost:9200/_snapshot/backup/?pretty
{
"backup" : {
"type" : "s3",
"settings" : {
"bucket" : "bucket",
"path_style_access" : "true",
"endpoint" : "https://storage",
"compress" : "true",
"max_snapshot_bytes_per_sec" : "10mb"
}
}
}
curl -s localhost:9200/_snapshot/backup/_all?pretty
{
"snapshots" : [
{
"snapshot" : "2019-07-22_00-17",
"uuid" : "gflODotoQJaSSJKl1X9x4w",
"version_id" : 5010299,
"version" : "5.1.2",
"indices" : [
"index_name"
],
"state" : "PARTIAL",
"start_time" : "2019-07-22T07:17:02.444Z",
"start_time_in_millis" : 1563779822444,
"end_time" : "2019-07-22T07:45:10.702Z",
"end_time_in_millis" : 1563781510702,
"duration_in_millis" : 1688258,
"failures" : [
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 92,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-0] during cleanup]; nested: NoSuchFileException[Blob [index-0] does not exist]; ",
"node_id" : "pPm9jafyTjyMk0T5A101xA",
"status" : "INTERNAL_SERVER_ERROR"
},
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 52,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-3] during cleanup]; nested: NoSuchFileException[Blob [index-3] does not exist]; ",
"node_id" : "pPm9jafyTjyMk0T5A101xA",
"status" : "INTERNAL_SERVER_ERROR"
},
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 60,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-0] during cleanup]; nested: NoSuchFileException[Blob [index-0] does not exist]; ",
"node_id" : "wKldjjxwTAa2-Y6A3Iz2xQ",
"status" : "INTERNAL_SERVER_ERROR"
},
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 51,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-4] during cleanup]; nested: NoSuchFileException[Blob [index-4] does not exist]; ",
"node_id" : "e3lycadTQMmlVotyf8Xoxw",
"status" : "INTERNAL_SERVER_ERROR"
},
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 64,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-439] during cleanup]; nested: NoSuchFileException[Blob [index-439] does not exist]; ",
"node_id" : "v8NlSfnLTDu0wDaDtHzTIg",
"status" : "INTERNAL_SERVER_ERROR"
},
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 50,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-7] during cleanup]; nested: NoSuchFileException[Blob [index-7] does not exist]; ",
"node_id" : "pPm9jafyTjyMk0T5A101xA",
"status" : "INTERNAL_SERVER_ERROR"
},
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 72,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-2] during cleanup]; nested: NoSuchFileException[Blob [index-2] does not exist]; ",
"node_id" : "v8NlSfnLTDu0wDaDtHzTIg",
"status" : "INTERNAL_SERVER_ERROR"
}
],
"shards" : {
"total" : 100,
"failed" : 7,
"successful" : 93
}
},
{
"snapshot" : "2019-07-22_03-17",
"uuid" : "ayITdAGFT72bpXdmz-nJNw",
"version_id" : 5010299,
"version" : "5.1.2",
"indices" : [
"index_name"
],
"state" : "PARTIAL",
"start_time" : "2019-07-22T10:17:03.152Z",
"start_time_in_millis" : 1563790623152,
"end_time" : "2019-07-22T10:44:18.938Z",
"end_time_in_millis" : 1563792258938,
"duration_in_millis" : 1635786,
"failures" : [
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 52,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [pending-index-5] during cleanup]; nested: NoSuchFileException[Blob [pending-index-5] does not exist]; ",
"node_id" : "pPm9jafyTjyMk0T5A101xA",
"status" : "INTERNAL_SERVER_ERROR"
},
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 64,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-440] during cleanup]; nested: NoSuchFileException[Blob [index-440] does not exist]; ",
"node_id" : "v8NlSfnLTDu0wDaDtHzTIg",
"status" : "INTERNAL_SERVER_ERROR"
},
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 92,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-1] during cleanup]; nested: NoSuchFileException[Blob [index-1] does not exist]; ",
"node_id" : "pPm9jafyTjyMk0T5A101xA",
"status" : "INTERNAL_SERVER_ERROR"
},
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 55,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-4] during cleanup]; nested: NoSuchFileException[Blob [index-4] does not exist]; ",
"node_id" : "v8NlSfnLTDu0wDaDtHzTIg",
"status" : "INTERNAL_SERVER_ERROR"
},
{
"index" : "index_name",
"index_uuid" : "index_name",
"shard_id" : 72,
"reason" : "IndexShardSnapshotFailedException[error deleting index file [index-0] during cleanup]; nested: NoSuchFileException[Blob [index-0] does not exist]; ",
"node_id" : "v8NlSfnLTDu0wDaDtHzTIg",
"status" : "INTERNAL_SERVER_ERROR"
}
],
"shards" : {
"total" : 100,
"failed" : 5,
"successful" : 95
}
},
{
"snapshot" : "2019-07-23_00-07",
"uuid" : "Qx5trg2uTSGuwB_DufbheA",
"version_id" : 5010299,
"version" : "5.1.2",
"indices" : [
"index_name"
],
"state" : "SUCCESS",
"start_time" : "2019-07-23T07:07:02.621Z",
"start_time_in_millis" : 1563865622621,
"end_time" : "2019-07-23T07:43:32.045Z",
"end_time_in_millis" : 1563867812045,
"duration_in_millis" : 2189424,
"failures" : [ ],
"shards" : {
"total" : 100,
"failed" : 0,
"successful" : 100
}
},
{
"snapshot" : "2019-07-24_00-14",
"uuid" : "vs-X86JzRhGiZlFoOp7oUg",
"version_id" : 5010299,
"version" : "5.1.2",
"indices" : [
"index_name"
],
"state" : "SUCCESS",
"start_time" : "2019-07-24T07:14:58.861Z",
"start_time_in_millis" : 1563952498861,
"end_time" : "2019-07-24T07:33:43.058Z",
"end_time_in_millis" : 1563953623058,
"duration_in_millis" : 1124197,
"failures" : [ ],
"shards" : {
"total" : 100,
"failed" : 0,
"successful" : 100
}
},
{
"snapshot" : "2019-07-25_00-13",
"uuid" : "JUAHRCIdQe6h2vaoqG3GRQ",
"version_id" : 5010299,
"version" : "5.1.2",
"indices" : [
"index_name"
],
"state" : "SUCCESS",
"start_time" : "2019-07-25T07:13:25.909Z",
"start_time_in_millis" : 1564038805909,
"end_time" : "2019-07-25T07:27:15.244Z",
"end_time_in_millis" : 1564039635244,
"duration_in_millis" : 829335,
"failures" : [ ],
"shards" : {
"total" : 100,
"failed" : 0,
"successful" : 100
}
},
{
"snapshot" : "2019-07-26_00-13",
"uuid" : "tmGFE6iLQJ6gph57zYpe3g",
"version_id" : 5010299,
"version" : "5.1.2",
"indices" : [
"index_name"
],
"state" : "SUCCESS",
"start_time" : "2019-07-26T07:13:50.068Z",
"start_time_in_millis" : 1564125230068,
"end_time" : "2019-07-26T07:34:25.833Z",
"end_time_in_millis" : 1564126465833,
"duration_in_millis" : 1235765,
"failures" : [ ],
"shards" : {
"total" : 100,
"failed" : 0,
"successful" : 100
}
},
{
"snapshot" : "2019-07-27_00-21",
"uuid" : "A8xl7gmfRS24vNa40kN0Sg",
"version_id" : 5010299,
"version" : "5.1.2",
"indices" : [
"index_name"
],
"state" : "SUCCESS",
"start_time" : "2019-07-27T07:21:17.092Z",
"start_time_in_millis" : 1564212077092,
"end_time" : "2019-07-27T07:39:05.425Z",
"end_time_in_millis" : 1564213145425,
"duration_in_millis" : 1068333,
"failures" : [ ],
"shards" : {
"total" : 100,
"failed" : 0,
"successful" : 100
}
},
{
"snapshot" : "2019-07-28_00-07",
"uuid" : "rkqfVf_kTLOcQkCVs3jsBQ",
"version_id" : 5010299,
"version" : "5.1.2",
"indices" : [
"index_name"
],
"state" : "SUCCESS",
"start_time" : "2019-07-28T07:07:02.667Z",
"start_time_in_millis" : 1564297622667,
"end_time" : "2019-07-28T07:47:43.388Z",
"end_time_in_millis" : 1564300063388,
"duration_in_millis" : 2440721,
"failures" : [ ],
"shards" : {
"total" : 100,
"failed" : 0,
"successful" : 100
}
},
{
"snapshot" : "2019-07-29_00-16",
"uuid" : "6Oc-0DZZRgCuVdYqR68C5Q",
"version_id" : 5010299,
"version" : "5.1.2",
"indices" : [
"index_name"
],
"state" : "SUCCESS",
"start_time" : "2019-07-29T07:16:43.188Z",
"start_time_in_millis" : 1564384603188,
"end_time" : "2019-07-29T07:23:34.874Z",
"end_time_in_millis" : 1564385014874,
"duration_in_millis" : 411686,
"failures" : [ ],
"shards" : {
"total" : 100,
"failed" : 0,
"successful" : 100
}
},
{
"snapshot" : "2019-07-30_00-07",
"uuid" : "sVg8PryoTUSRSON9Rypl9w",
"version_id" : 5010299,
"version" : "5.1.2",
"indices" : [
"index_name"
],
"state" : "SUCCESS",
"start_time" : "2019-07-30T07:07:01.490Z",
"start_time_in_millis" : 1564470421490,
"end_time" : "2019-07-30T07:25:27.493Z",
"end_time_in_millis" : 1564471527493,
"duration_in_millis" : 1106003,
"failures" : [ ],
"shards" : {
"total" : 100,
"failed" : 0,
"successful" : 100
}
}
]
}What could be the problem?