手动运行降采样
编辑手动运行降采样编辑
降采样时间序列数据流 (TSDS) 的推荐方法是通过索引生命周期管理 (ILM)。但是，如果您没有使用 ILM，则可以手动降采样 TSDS。本指南将向您展示如何使用典型的 Kubernetes 集群监控数据来完成此操作。
要测试手动降采样，请执行以下步骤
先决条件编辑
- 请参阅TSDS 先决条件。
- 无法直接对数据流进行降采样，也无法一次对多个索引进行降采样。只能对一个时间序列索引（TSDS 支持索引）进行降采样。
- 为了对索引进行降采样，它需要是只读的。对于 TSDS 写入索引，这意味着需要先对其进行滚动并使其变为只读。
- 降采样使用 UTC 时间戳。
- 降采样需要在时间序列索引中至少存在一个指标字段。
创建时间序列数据流编辑
首先，您将创建一个 TSDS。为简单起见，在时间序列映射中，所有 time_series_metric 参数都设置为类型 gauge，但其他值（例如 counter 和 histogram）也可以使用。 time_series_metric 值决定了在降采样期间使用的统计表示类型。
索引模板包含一组静态时间序列维度：host、namespace、node 和 pod。时间序列维度不会被降采样过程改变。
response = client.indices.put_index_template(
name: 'my-data-stream-template',
body: {
index_patterns: [
'my-data-stream*'
],
data_stream: {},
template: {
settings: {
index: {
mode: 'time_series',
routing_path: [
'kubernetes.namespace',
'kubernetes.host',
'kubernetes.node',
'kubernetes.pod'
],
number_of_replicas: 0,
number_of_shards: 2
}
},
mappings: {
properties: {
"@timestamp": {
type: 'date'
},
kubernetes: {
properties: {
container: {
properties: {
cpu: {
properties: {
usage: {
properties: {
core: {
properties: {
ns: {
type: 'long'
}
}
},
limit: {
properties: {
pct: {
type: 'float'
}
}
},
nanocores: {
type: 'long',
time_series_metric: 'gauge'
},
node: {
properties: {
pct: {
type: 'float'
}
}
}
}
}
}
},
memory: {
properties: {
available: {
properties: {
bytes: {
type: 'long',
time_series_metric: 'gauge'
}
}
},
majorpagefaults: {
type: 'long'
},
pagefaults: {
type: 'long',
time_series_metric: 'gauge'
},
rss: {
properties: {
bytes: {
type: 'long',
time_series_metric: 'gauge'
}
}
},
usage: {
properties: {
bytes: {
type: 'long',
time_series_metric: 'gauge'
},
limit: {
properties: {
pct: {
type: 'float'
}
}
},
node: {
properties: {
pct: {
type: 'float'
}
}
}
}
},
workingset: {
properties: {
bytes: {
type: 'long',
time_series_metric: 'gauge'
}
}
}
}
},
name: {
type: 'keyword'
},
start_time: {
type: 'date'
}
}
},
host: {
type: 'keyword',
time_series_dimension: true
},
namespace: {
type: 'keyword',
time_series_dimension: true
},
node: {
type: 'keyword',
time_series_dimension: true
},
pod: {
type: 'keyword',
time_series_dimension: true
}
}
}
}
}
}
}
)
puts response
PUT _index_template/my-data-stream-template
{
"index_patterns": [
"my-data-stream*"
],
"data_stream": {},
"template": {
"settings": {
"index": {
"mode": "time_series",
"routing_path": [
"kubernetes.namespace",
"kubernetes.host",
"kubernetes.node",
"kubernetes.pod"
],
"number_of_replicas": 0,
"number_of_shards": 2
}
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"kubernetes": {
"properties": {
"container": {
"properties": {
"cpu": {
"properties": {
"usage": {
"properties": {
"core": {
"properties": {
"ns": {
"type": "long"
}
}
},
"limit": {
"properties": {
"pct": {
"type": "float"
}
}
},
"nanocores": {
"type": "long",
"time_series_metric": "gauge"
},
"node": {
"properties": {
"pct": {
"type": "float"
}
}
}
}
}
}
},
"memory": {
"properties": {
"available": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
},
"majorpagefaults": {
"type": "long"
},
"pagefaults": {
"type": "long",
"time_series_metric": "gauge"
},
"rss": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
},
"usage": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
},
"limit": {
"properties": {
"pct": {
"type": "float"
}
}
},
"node": {
"properties": {
"pct": {
"type": "float"
}
}
}
}
},
"workingset": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
}
}
},
"name": {
"type": "keyword"
},
"start_time": {
"type": "date"
}
}
},
"host": {
"type": "keyword",
"time_series_dimension": true
},
"namespace": {
"type": "keyword",
"time_series_dimension": true
},
"node": {
"type": "keyword",
"time_series_dimension": true
},
"pod": {
"type": "keyword",
"time_series_dimension": true
}
}
}
}
}
}
}
提取时间序列数据编辑
因为时间序列数据流被设计为仅接受最近的数据，在本例中，您将使用提取管道在数据被索引时对其进行时间偏移。因此，索引数据的 @timestamp 将来自过去 15 分钟。
使用以下请求创建管道
response = client.ingest.put_pipeline(
id: 'my-timestamp-pipeline',
body: {
description: 'Shifts the @timestamp to the last 15 minutes',
processors: [
{
set: {
field: 'ingest_time',
value: '{{_ingest.timestamp}}'
}
},
{
script: {
lang: 'painless',
source: "\n def delta = ChronoUnit.SECONDS.between(\n ZonedDateTime.parse(\"2022-06-21T15:49:00Z\"),\n ZonedDateTime.parse(ctx[\"ingest_time\"])\n );\n ctx[\"@timestamp\"] = ZonedDateTime.parse(ctx[\"@timestamp\"]).plus(delta,ChronoUnit.SECONDS).toString();\n "
}
}
]
}
)
puts response
PUT _ingest/pipeline/my-timestamp-pipeline
{
"description": "Shifts the @timestamp to the last 15 minutes",
"processors": [
{
"set": {
"field": "ingest_time",
"value": "{{_ingest.timestamp}}"
}
},
{
"script": {
"lang": "painless",
"source": """
def delta = ChronoUnit.SECONDS.between(
ZonedDateTime.parse("2022-06-21T15:49:00Z"),
ZonedDateTime.parse(ctx["ingest_time"])
);
ctx["@timestamp"] = ZonedDateTime.parse(ctx["@timestamp"]).plus(delta,ChronoUnit.SECONDS).toString();
"""
}
}
]
}
接下来，使用批量 API 请求自动创建您的 TSDS 并索引一组十个文档
response = client.bulk(
index: 'my-data-stream',
refresh: true,
pipeline: 'my-timestamp-pipeline',
body: [
{
create: {}
},
{
"@timestamp": '2022-06-21T15:49:00Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 91_153,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 463_314_616
},
usage: {
bytes: 307_007_078,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 585_236
},
rss: {
bytes: 102_728
},
pagefaults: 120_901,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:45:50Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 124_501,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 982_546_514
},
usage: {
bytes: 360_035_574,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_339_884
},
rss: {
bytes: 381_174
},
pagefaults: 178_473,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:44:50Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 38_907,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 862_723_768
},
usage: {
bytes: 379_572_388,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 431_227
},
rss: {
bytes: 386_580
},
pagefaults: 233_166,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:44:40Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 86_706,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 567_160_996
},
usage: {
bytes: 103_266_017,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_724_908
},
rss: {
bytes: 105_431
},
pagefaults: 233_166,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:44:00Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 150_069,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 639_054_643
},
usage: {
bytes: 265_142_477,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_786_511
},
rss: {
bytes: 189_235
},
pagefaults: 138_172,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:42:40Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 82_260,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 854_735_585
},
usage: {
bytes: 309_798_052,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 924_058
},
rss: {
bytes: 110_838
},
pagefaults: 259_073,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:42:10Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 153_404,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 279_586_406
},
usage: {
bytes: 214_904_955,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_047_265
},
rss: {
bytes: 91_914
},
pagefaults: 302_252,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:40:20Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 125_613,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 822_782_853
},
usage: {
bytes: 100_475_044,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 2_109_932
},
rss: {
bytes: 278_446
},
pagefaults: 74_843,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:40:10Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 100_046,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 567_160_996
},
usage: {
bytes: 362_826_547,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_986_724
},
rss: {
bytes: 402_801
},
pagefaults: 296_495,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:38:30Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 40_018,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 1_062_428_344
},
usage: {
bytes: 265_142_477,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 2_294_743
},
rss: {
bytes: 340_623
},
pagefaults: 224_530,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
}
]
)
puts response
PUT /my-data-stream/_bulk?refresh&pipeline=my-timestamp-pipeline
{"create": {}}
{"@timestamp":"2022-06-21T15:49:00Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":91153,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":463314616},"usage":{"bytes":307007078,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":585236},"rss":{"bytes":102728},"pagefaults":120901,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:45:50Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":124501,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":982546514},"usage":{"bytes":360035574,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1339884},"rss":{"bytes":381174},"pagefaults":178473,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:44:50Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":38907,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":862723768},"usage":{"bytes":379572388,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":431227},"rss":{"bytes":386580},"pagefaults":233166,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:44:40Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":86706,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":567160996},"usage":{"bytes":103266017,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1724908},"rss":{"bytes":105431},"pagefaults":233166,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:44:00Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":150069,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":639054643},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1786511},"rss":{"bytes":189235},"pagefaults":138172,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:42:40Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":82260,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":854735585},"usage":{"bytes":309798052,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":924058},"rss":{"bytes":110838},"pagefaults":259073,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:42:10Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":153404,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":279586406},"usage":{"bytes":214904955,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1047265},"rss":{"bytes":91914},"pagefaults":302252,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:40:20Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":125613,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":822782853},"usage":{"bytes":100475044,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2109932},"rss":{"bytes":278446},"pagefaults":74843,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:40:10Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":100046,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":567160996},"usage":{"bytes":362826547,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1986724},"rss":{"bytes":402801},"pagefaults":296495,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:38:30Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":40018,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":1062428344},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2294743},"rss":{"bytes":340623},"pagefaults":224530,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
您可以使用搜索 API 检查文档是否已正确索引
response = client.search( index: 'my-data-stream' ) puts response
GET /my-data-stream/_search
对数据运行以下聚合以计算一些有趣的统计数据
response = client.search(
index: 'my-data-stream',
body: {
size: 0,
aggregations: {
tsid: {
terms: {
field: '_tsid'
},
aggregations: {
over_time: {
date_histogram: {
field: '@timestamp',
fixed_interval: '1d'
},
aggregations: {
min: {
min: {
field: 'kubernetes.container.memory.usage.bytes'
}
},
max: {
max: {
field: 'kubernetes.container.memory.usage.bytes'
}
},
avg: {
avg: {
field: 'kubernetes.container.memory.usage.bytes'
}
}
}
}
}
}
}
}
)
puts response
GET /my-data-stream/_search
{
"size": 0,
"aggs": {
"tsid": {
"terms": {
"field": "_tsid"
},
"aggs": {
"over_time": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "1d"
},
"aggs": {
"min": {
"min": {
"field": "kubernetes.container.memory.usage.bytes"
}
},
"max": {
"max": {
"field": "kubernetes.container.memory.usage.bytes"
}
},
"avg": {
"avg": {
"field": "kubernetes.container.memory.usage.bytes"
}
}
}
}
}
}
}
}
降采样 TSDS编辑
无法直接对 TSDS 进行降采样。您需要对其支持索引进行降采样。您可以通过运行以下命令来查看数据流的支持索引
response = client.indices.get_data_stream( name: 'my-data-stream' ) puts response
GET /_data_stream/my-data-stream
这将返回
{
"data_streams": [
{
"name": "my-data-stream",
"timestamp_field": {
"name": "@timestamp"
},
"indices": [
{
"index_name": ".ds-my-data-stream-2023.07.26-000001",
"index_uuid": "ltOJGmqgTVm4T-Buoe7Acg",
"prefer_ilm": true,
"managed_by": "Unmanaged"
}
],
"generation": 1,
"status": "GREEN",
"next_generation_managed_by": "Unmanaged",
"prefer_ilm": true,
"template": "my-data-stream-template",
"hidden": false,
"system": false,
"allow_custom_routing": false,
"replicated": false,
"rollover_on_write": false,
"time_series": {
"temporal_ranges": [
{
"start": "2023-07-26T09:26:42.000Z",
"end": "2023-07-26T13:26:42.000Z"
}
]
}
}
]
}
在对支持索引进行降采样之前，需要对 TSDS 进行滚动并将旧索引设置为只读。
使用滚动 API滚动 TSDS
response = client.indices.rollover( alias: 'my-data-stream' ) puts response
POST /my-data-stream/_rollover/
从响应中复制 old_index 的名称。在以下步骤中，将索引名称替换为您的 old_index 的名称。
需要将旧索引设置为只读模式。运行以下请求
response = client.indices.add_block( index: '.ds-my-data-stream-2023.07.26-000001', block: 'write' ) puts response
PUT /.ds-my-data-stream-2023.07.26-000001/_block/write
接下来，使用降采样 API对索引进行降采样，将时间序列间隔设置为一小时
response = client.indices.downsample(
index: '.ds-my-data-stream-2023.07.26-000001',
target_index: '.ds-my-data-stream-2023.07.26-000001-downsample',
body: {
fixed_interval: '1h'
}
)
puts response
POST /.ds-my-data-stream-2023.07.26-000001/_downsample/.ds-my-data-stream-2023.07.26-000001-downsample
{
"fixed_interval": "1h"
}
现在您可以修改数据流，并将原始索引替换为降采样后的索引
POST _data_stream/_modify
{
"actions": [
{
"remove_backing_index": {
"data_stream": "my-data-stream",
"index": ".ds-my-data-stream-2023.07.26-000001"
}
},
{
"add_backing_index": {
"data_stream": "my-data-stream",
"index": ".ds-my-data-stream-2023.07.26-000001-downsample"
}
}
]
}
您现在可以删除旧的支持索引。但请注意，这将删除原始数据。如果您将来可能需要原始数据，请不要删除该索引。
查看结果编辑
重新运行之前的搜索查询（请注意，在查询降采样索引时，有一些细微差别需要注意）
response = client.search( index: 'my-data-stream' ) puts response
GET /my-data-stream/_search
具有新的降采样支持索引的 TSDS 仅包含一个文档。对于计数器，此文档将仅具有最后一个值。对于仪表，字段类型现在是 aggregate_metric_double。您将看到基于原始采样指标的 min、max、sum 和 value_count 统计信息
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 4,
"successful": 4,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": ".ds-my-data-stream-2023.07.26-000001-downsample",
"_id": "0eL0wC_4-45SnTNFAAABiZHbD4A",
"_score": 1,
"_source": {
"@timestamp": "2023-07-26T11:00:00.000Z",
"_doc_count": 10,
"ingest_time": "2023-07-26T11:26:42.715Z",
"kubernetes": {
"container": {
"cpu": {
"usage": {
"core": {
"ns": 12828317850
},
"limit": {
"pct": 0.0000277905
},
"nanocores": {
"min": 38907,
"max": 153404,
"sum": 992677,
"value_count": 10
},
"node": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": {
"min": 279586406,
"max": 1062428344,
"sum": 7101494721,
"value_count": 10
}
},
"majorpagefaults": 0,
"pagefaults": {
"min": 74843,
"max": 302252,
"sum": 2061071,
"value_count": 10
},
"rss": {
"bytes": {
"min": 91914,
"max": 402801,
"sum": 2389770,
"value_count": 10
}
},
"usage": {
"bytes": {
"min": 100475044,
"max": 379572388,
"sum": 2668170609,
"value_count": 10
},
"limit": {
"pct": 0.00009923134
},
"node": {
"pct": 0.017700378
}
},
"workingset": {
"bytes": {
"min": 431227,
"max": 2294743,
"sum": 14230488,
"value_count": 10
}
}
},
"name": "container-name-44",
"start_time": "2021-03-30T07:59:06.000Z"
},
"host": "gke-apps-0",
"namespace": "namespace26",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0"
}
}
}
]
}
}
重新运行之前的聚合。即使聚合是在仅包含 1 个文档的降采样 TSDS 上运行的，它也会返回与之前在原始 TSDS 上运行的聚合相同的结果。
response = client.search(
index: 'my-data-stream',
body: {
size: 0,
aggregations: {
tsid: {
terms: {
field: '_tsid'
},
aggregations: {
over_time: {
date_histogram: {
field: '@timestamp',
fixed_interval: '1d'
},
aggregations: {
min: {
min: {
field: 'kubernetes.container.memory.usage.bytes'
}
},
max: {
max: {
field: 'kubernetes.container.memory.usage.bytes'
}
},
avg: {
avg: {
field: 'kubernetes.container.memory.usage.bytes'
}
}
}
}
}
}
}
}
)
puts response
GET /my-data-stream/_search
{
"size": 0,
"aggs": {
"tsid": {
"terms": {
"field": "_tsid"
},
"aggs": {
"over_time": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "1d"
},
"aggs": {
"min": {
"min": {
"field": "kubernetes.container.memory.usage.bytes"
}
},
"max": {
"max": {
"field": "kubernetes.container.memory.usage.bytes"
}
},
"avg": {
"avg": {
"field": "kubernetes.container.memory.usage.bytes"
}
}
}
}
}
}
}
}
此示例演示了降采样如何在您选择的任何时间范围内显着减少为时间序列数据存储的文档数量。也可以对已经降采样的数据执行降采样，以随着时间序列数据的老化和数据分辨率变得不那么重要而进一步减少存储和相关成本。
降采样 TSDS 的推荐方法是使用 ILM。要了解更多信息，请尝试使用 ILM 运行降采样示例。